From 8fa6df940afd0dde38174a27dcdb3861acccbc82 Mon Sep 17 00:00:00 2001 From: tdruez Date: Mon, 10 Jun 2024 09:02:05 +0400 Subject: [PATCH 01/27] Add new license fields on Component and Package models #63 Signed-off-by: tdruez --- .../migrations/0005_license_fields.py | 133 ++++++++++++++++++ component_catalog/models.py | 66 +++++++++ 2 files changed, 199 insertions(+) create mode 100644 component_catalog/migrations/0005_license_fields.py diff --git a/component_catalog/migrations/0005_license_fields.py b/component_catalog/migrations/0005_license_fields.py new file mode 100644 index 00000000..add29dcc --- /dev/null +++ b/component_catalog/migrations/0005_license_fields.py @@ -0,0 +1,133 @@ +# Generated by Django 5.0.6 on 2024-06-10 05:01 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("component_catalog", "0004_initial"), + ] + + operations = [ + migrations.AddField( + model_name="component", + name="declared_license_expression", + field=models.TextField( + blank=True, + help_text="A license expression derived from statements in the key files of a software project, such as the NOTICE, COPYING, README, and LICENSE files.", + ), + ), + migrations.AddField( + model_name="component", + name="declared_license_expression_spdx", + field=models.TextField( + blank=True, + help_text="A declared license expression that uses the license identifiers defined by SPDX, as well as the “Licenseref” syntax for licenses not on the SPDX list.", + verbose_name="Declared license expression SPDX", + ), + ), + migrations.AddField( + model_name="component", + name="extracted_license_statement", + field=models.TextField( + blank=True, + help_text="The actual text extracted from a software project that supports the license detection process.", + ), + ), + migrations.AddField( + model_name="component", + name="license_detections", + field=models.JSONField( + blank=True, + default=list, + help_text="A list of specific license identifiers derived from statements in the key files of a software project, such as the NOTICE, COPYING, README, and LICENSE files.", + ), + ), + migrations.AddField( + model_name="component", + name="other_license_detections", + field=models.JSONField( + blank=True, + default=list, + help_text="A list of specific license identifiers derived from the non-key files of a software project, which are very often third-party software used by the project, or test, sample and documentation files.", + ), + ), + migrations.AddField( + model_name="component", + name="other_license_expression", + field=models.TextField( + blank=True, + help_text="A license expression derived from detected licenses in the non-key files of a software project, which are often third-party software used by the project, or test, sample and documentation files.", + ), + ), + migrations.AddField( + model_name="component", + name="other_license_expression_spdx", + field=models.TextField( + blank=True, + help_text="A license expression derived from detected licenses in the non-key files of a software project, using the license identifiers defined by SPDX, as well as the “Licenseref” syntax for licenses not on the SPDX list.", + verbose_name="Other license expression SPDX", + ), + ), + migrations.AddField( + model_name="package", + name="declared_license_expression", + field=models.TextField( + blank=True, + help_text="A license expression derived from statements in the key files of a software project, such as the NOTICE, COPYING, README, and LICENSE files.", + ), + ), + migrations.AddField( + model_name="package", + name="declared_license_expression_spdx", + field=models.TextField( + blank=True, + help_text="A declared license expression that uses the license identifiers defined by SPDX, as well as the “Licenseref” syntax for licenses not on the SPDX list.", + verbose_name="Declared license expression SPDX", + ), + ), + migrations.AddField( + model_name="package", + name="extracted_license_statement", + field=models.TextField( + blank=True, + help_text="The actual text extracted from a software project that supports the license detection process.", + ), + ), + migrations.AddField( + model_name="package", + name="license_detections", + field=models.JSONField( + blank=True, + default=list, + help_text="A list of specific license identifiers derived from statements in the key files of a software project, such as the NOTICE, COPYING, README, and LICENSE files.", + ), + ), + migrations.AddField( + model_name="package", + name="other_license_detections", + field=models.JSONField( + blank=True, + default=list, + help_text="A list of specific license identifiers derived from the non-key files of a software project, which are very often third-party software used by the project, or test, sample and documentation files.", + ), + ), + migrations.AddField( + model_name="package", + name="other_license_expression", + field=models.TextField( + blank=True, + help_text="A license expression derived from detected licenses in the non-key files of a software project, which are often third-party software used by the project, or test, sample and documentation files.", + ), + ), + migrations.AddField( + model_name="package", + name="other_license_expression_spdx", + field=models.TextField( + blank=True, + help_text="A license expression derived from detected licenses in the non-key files of a software project, using the license identifiers defined by SPDX, as well as the “Licenseref” syntax for licenses not on the SPDX list.", + verbose_name="Other license expression SPDX", + ), + ), + ] diff --git a/component_catalog/models.py b/component_catalog/models.py index e4917f4c..465f0d41 100644 --- a/component_catalog/models.py +++ b/component_catalog/models.py @@ -294,6 +294,70 @@ def compliance_table_class(self): return "table-warning" +class LicenseFieldsMixin(models.Model): + declared_license_expression = models.TextField( + blank=True, + help_text=_( + "A license expression derived from statements in the key files of a " + "software project, such as the NOTICE, COPYING, README, and LICENSE files." + ), + ) + declared_license_expression_spdx = models.TextField( + verbose_name=_("Declared license expression SPDX"), + blank=True, + help_text=_( + "A declared license expression that uses the license identifiers defined " + "by SPDX, as well as the “Licenseref” syntax for licenses not on the SPDX " + "list." + ), + ) + license_detections = models.JSONField( + default=list, + blank=True, + help_text=_( + "A list of specific license identifiers derived from statements in the " + "key files of a software project, such as the NOTICE, COPYING, README, " + "and LICENSE files." + ), + ) + other_license_expression = models.TextField( + blank=True, + help_text=_( + "A license expression derived from detected licenses in the non-key files " + "of a software project, which are often third-party software used by the " + "project, or test, sample and documentation files." + ), + ) + other_license_expression_spdx = models.TextField( + verbose_name=_("Other license expression SPDX"), + blank=True, + help_text=_( + "A license expression derived from detected licenses in the non-key files " + "of a software project, using the license identifiers defined by SPDX, " + "as well as the “Licenseref” syntax for licenses not on the SPDX list." + ), + ) + other_license_detections = models.JSONField( + default=list, + blank=True, + help_text=_( + "A list of specific license identifiers derived from the non-key files " + "of a software project, which are very often third-party software used " + "by the project, or test, sample and documentation files." + ), + ) + extracted_license_statement = models.TextField( + blank=True, + help_text=_( + "The actual text extracted from a software project that supports the " + "license detection process." + ), + ) + + class Meta: + abstract = True + + def get_cyclonedx_properties(instance): """ Return fields not supported natively by CycloneDX as properties. @@ -819,6 +883,7 @@ class Component( HolderMixin, KeywordsMixin, CPEMixin, + LicenseFieldsMixin, ParentChildModelMixin, BaseComponentMixin, DataspacedModel, @@ -1584,6 +1649,7 @@ class Package( UsagePolicyMixin, SetPolicyFromLicenseMixin, LicenseExpressionMixin, + LicenseFieldsMixin, RequestMixin, HistoryFieldsMixin, ReferenceNotesMixin, From f8510581cb9feff0e83cf9201822180ef082b470 Mon Sep 17 00:00:00 2001 From: tdruez Date: Mon, 10 Jun 2024 11:04:10 +0400 Subject: [PATCH 02/27] Add new license fields in UI and related features #63 Signed-off-by: tdruez --- component_catalog/admin.py | 14 +++- component_catalog/api.py | 28 +++++++ .../migrations/0005_license_fields.py | 24 +++++- component_catalog/models.py | 59 +++++++-------- component_catalog/views.py | 18 ++++- .../migrations/0005_license_fields.py | 45 +++++++++++ product_portfolio/models.py | 74 ++++++++++--------- .../product_portfolio/tabs/tab_inventory.html | 2 +- 8 files changed, 195 insertions(+), 69 deletions(-) create mode 100644 product_portfolio/migrations/0005_license_fields.py diff --git a/component_catalog/admin.py b/component_catalog/admin.py index 6f745659..b74a96f2 100644 --- a/component_catalog/admin.py +++ b/component_catalog/admin.py @@ -826,6 +826,13 @@ class PackageAdmin( { "fields": ( "license_expression", + "declared_license_expression", + "declared_license_expression_spdx", + "license_detections", + "other_license_expression", + "other_license_expression_spdx", + "other_license_detections", + "extracted_license_statement", "copyright", "holder", "author", @@ -880,7 +887,12 @@ class PackageAdmin( ), get_additional_information_fieldset(), ] - readonly_fields = DataspacedAdmin.readonly_fields + ("package_url", "inferred_url") + readonly_fields = DataspacedAdmin.readonly_fields + ( + "package_url", + "inferred_url", + "declared_license_expression_spdx", + "other_license_expression_spdx", + ) form = PackageAdminForm importer_class = PackageImporter mass_update_form = PackageMassUpdateForm diff --git a/component_catalog/api.py b/component_catalog/api.py index e9ad4fbf..31cd4de7 100644 --- a/component_catalog/api.py +++ b/component_catalog/api.py @@ -140,6 +140,13 @@ class Meta: "holder", "author", "license_expression", + "declared_license_expression", + "declared_license_expression_spdx", + "license_detections", + "other_license_expression", + "other_license_expression_spdx", + "other_license_detections", + "extracted_license_statement", "reference_notes", "homepage_url", "vcs_url", @@ -323,6 +330,13 @@ class Meta: "licenses_summary", "license_choices_expression", "license_choices", + "declared_license_expression", + "declared_license_expression_spdx", + "license_detections", + "other_license_expression", + "other_license_expression_spdx", + "other_license_detections", + "extracted_license_statement", "created_date", "last_modified_date", ) @@ -524,6 +538,13 @@ class Meta(ComponentSerializer.Meta): "copyright", "holder", "license_expression", + "declared_license_expression", + "declared_license_expression_spdx", + "license_detections", + "other_license_expression", + "other_license_expression_spdx", + "other_license_detections", + "extracted_license_statement", "reference_notes", "release_date", "description", @@ -639,6 +660,13 @@ class Meta: "licenses_summary", "license_choices_expression", "license_choices", + "declared_license_expression", + "declared_license_expression_spdx", + "license_detections", + "other_license_expression", + "other_license_expression_spdx", + "other_license_detections", + "extracted_license_statement", "reference_notes", "homepage_url", "vcs_url", diff --git a/component_catalog/migrations/0005_license_fields.py b/component_catalog/migrations/0005_license_fields.py index add29dcc..d448a8e3 100644 --- a/component_catalog/migrations/0005_license_fields.py +++ b/component_catalog/migrations/0005_license_fields.py @@ -1,4 +1,4 @@ -# Generated by Django 5.0.6 on 2024-06-10 05:01 +# Generated by Django 5.0.6 on 2024-06-10 06:37 from django.db import migrations, models @@ -130,4 +130,26 @@ class Migration(migrations.Migration): verbose_name="Other license expression SPDX", ), ), + migrations.AlterField( + model_name="component", + name="license_expression", + field=models.CharField( + blank=True, + db_index=True, + help_text='The License Expression assigned to a DejaCode Package or Component is an editable value equivalent to a "concluded license" as determined by a curator who has performed analysis to clarify or correct the declared license expression, which may have been assigned automatically (from a scan or an associated package definition) when the Package or Component was originally created. A license expression defines the relationship of one or more licenses to a software object. More than one applicable license can be expressed as "license-key-a AND license-key-b". A choice of applicable licenses can be expressed as "license-key-a OR license-key-b", and you can indicate the primary (preferred) license by placing it first, on the left-hand side of the OR relationship. The relationship words (OR, AND) can be combined as needed, and the use of parentheses can be applied to clarify the meaning; for example "((license-key-a AND license-key-b) OR (license-key-c))". An exception to a license can be expressed as "license-key WITH license-exception-key".', + max_length=1024, + verbose_name="Concluded license expression", + ), + ), + migrations.AlterField( + model_name="package", + name="license_expression", + field=models.CharField( + blank=True, + db_index=True, + help_text='The License Expression assigned to a DejaCode Package or Component is an editable value equivalent to a "concluded license" as determined by a curator who has performed analysis to clarify or correct the declared license expression, which may have been assigned automatically (from a scan or an associated package definition) when the Package or Component was originally created. A license expression defines the relationship of one or more licenses to a software object. More than one applicable license can be expressed as "license-key-a AND license-key-b". A choice of applicable licenses can be expressed as "license-key-a OR license-key-b", and you can indicate the primary (preferred) license by placing it first, on the left-hand side of the OR relationship. The relationship words (OR, AND) can be combined as needed, and the use of parentheses can be applied to clarify the meaning; for example "((license-key-a AND license-key-b) OR (license-key-c))". An exception to a license can be expressed as "license-key WITH license-exception-key".', + max_length=1024, + verbose_name="Concluded license expression", + ), + ), ] diff --git a/component_catalog/models.py b/component_catalog/models.py index 465f0d41..630006d0 100644 --- a/component_catalog/models.py +++ b/component_catalog/models.py @@ -92,6 +92,24 @@ "version", ] +LICENSE_EXPRESSION_HELP_TEXT = _( + "The License Expression assigned to a DejaCode Package or Component is an editable " + 'value equivalent to a "concluded license" as determined by a curator who has ' + "performed analysis to clarify or correct the declared license expression, which " + "may have been assigned automatically (from a scan or an associated package " + "definition) when the Package or Component was originally created. " + "A license expression defines the relationship of one or more licenses to a " + "software object. More than one applicable license can be expressed as " + '"license-key-a AND license-key-b". A choice of applicable licenses can be ' + 'expressed as "license-key-a OR license-key-b", and you can indicate the primary ' + "(preferred) license by placing it first, on the left-hand side of the OR " + "relationship. The relationship words (OR, AND) can be combined as needed, " + "and the use of parentheses can be applied to clarify the meaning; " + 'for example "((license-key-a AND license-key-b) OR (license-key-c))". ' + "An exception to a license can be expressed as " + '"license-key WITH license-exception-key".' +) + def validate_filename(value): invalid_chars = ["/", "\\", ":"] @@ -755,25 +773,6 @@ class BaseComponentMixin( ), ) - license_expression = models.CharField( - _("License expression"), - max_length=1024, - blank=True, - db_index=True, - help_text=_( - "On a component or a product in DejaCode, a license expression defines the " - "relationship of one or more licenses to that software as declared by its " - 'licensor. More than one applicable license can be expressed as "license-key-a ' - 'AND license-key-b". A choice of applicable licenses can be expressed as ' - '"license-key-a OR license-key-b", and you can indicate the primary (preferred) ' - "license by placing it first, on the left-hand side of the OR relationship. " - "The relationship words (OR, AND) can be combined as needed, and the use of " - 'parentheses can be applied to clarify the meaning; for example "((license-key-a ' - 'AND license-key-b) OR (license-key-c))". An exception to a license can be ' - 'expressed as “license-key WITH license-exception-key".' - ), - ) - class Meta: abstract = True unique_together = (("dataspace", "name", "version"), ("dataspace", "uuid")) @@ -888,6 +887,14 @@ class Component( BaseComponentMixin, DataspacedModel, ): + license_expression = models.CharField( + _("Concluded license expression"), + max_length=1024, + blank=True, + db_index=True, + help_text=LICENSE_EXPRESSION_HELP_TEXT, + ) + configuration_status = models.ForeignKey( to="component_catalog.ComponentStatus", on_delete=models.PROTECT, @@ -1738,21 +1745,11 @@ class Package( ) license_expression = models.CharField( - _("License expression"), + _("Concluded license expression"), max_length=1024, blank=True, db_index=True, - help_text=_( - "On a package in DejaCode, a license expression defines the relationship of one or " - "more licenses to that software as declared by its licensor. More than one " - 'applicable license can be expressed as "license-key-a AND license-key-b". A choice ' - 'of applicable licenses can be expressed as "license-key-a OR license-key-b", and you ' - "can indicate the primary (preferred) license by placing it first, on the left-hand " - "side of the OR relationship. The relationship words (OR, AND) can be combined as " - "needed, and the use of parentheses can be applied to clarify the meaning; for " - 'example "((license-key-a AND license-key-b) OR (license-key-c))". An exception to ' - 'a license can be expressed as “license-key WITH license-exception-key".' - ), + help_text=LICENSE_EXPRESSION_HELP_TEXT, ) copyright = models.TextField( diff --git a/component_catalog/views.py b/component_catalog/views.py index 943eb8cf..bc8d1f1c 100644 --- a/component_catalog/views.py +++ b/component_catalog/views.py @@ -435,7 +435,7 @@ class ComponentListView( Header("name", _("Component name")), Header("version", _("Version")), Header("usage_policy", _("Policy"), filter="usage_policy", condition=include_policy), - Header("license_expression", _("License"), filter="licenses"), + Header("license_expression", _("Concluded license"), filter="licenses"), Header("primary_language", _("Language"), filter="primary_language"), Header("owner", _("Owner")), Header("keywords", _("Keywords"), filter="keywords"), @@ -562,6 +562,13 @@ class ComponentDetailsView( "reference_notes", "licenses", "licenses_summary", + "declared_license_expression", + "declared_license_expression_spdx", + "license_detections", + "other_license_expression", + "other_license_expression_spdx", + "other_license_detections", + "extracted_license_statement", ], }, "hierarchy": {}, @@ -1073,7 +1080,7 @@ class PackageListView( table_headers = ( Header("sortable_identifier", _("Identifier"), Package.identifier_help()), Header("usage_policy", _("Policy"), filter="usage_policy", condition=include_policy), - Header("license_expression", _("License"), filter="licenses"), + Header("license_expression", _("Concluded license"), filter="licenses"), Header("primary_language", _("Language"), filter="primary_language"), Header("filename", _("Download"), help_text="Download link"), Header("components", "Components", PACKAGE_COMPONENTS_HELP, "component"), @@ -1214,6 +1221,13 @@ class PackageDetailsView( "reference_notes", "licenses", "licenses_summary", + "declared_license_expression", + "declared_license_expression_spdx", + "license_detections", + "other_license_expression", + "other_license_expression_spdx", + "other_license_detections", + "extracted_license_statement", ], }, "terms": { diff --git a/product_portfolio/migrations/0005_license_fields.py b/product_portfolio/migrations/0005_license_fields.py new file mode 100644 index 00000000..9bb0b528 --- /dev/null +++ b/product_portfolio/migrations/0005_license_fields.py @@ -0,0 +1,45 @@ +# Generated by Django 5.0.6 on 2024-06-10 06:37 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("product_portfolio", "0004_alter_scancodeproject_type"), + ] + + operations = [ + migrations.AlterField( + model_name="product", + name="license_expression", + field=models.CharField( + blank=True, + db_index=True, + help_text='On a product in DejaCode, a license expression defines the relationship of one or more licenses to that software as declared by its licensor. More than one applicable license can be expressed as "license-key-a AND license-key-b". A choice of applicable licenses can be expressed as "license-key-a OR license-key-b", and you can indicate the primary (preferred) license by placing it first, on the left-hand side of the OR relationship. The relationship words (OR, AND) can be combined as needed, and the use of parentheses can be applied to clarify the meaning; for example "((license-key-a AND license-key-b) OR (license-key-c))". An exception to a license can be expressed as "license-key WITH license-exception-key".', + max_length=1024, + ), + ), + migrations.AlterField( + model_name="productcomponent", + name="license_expression", + field=models.CharField( + blank=True, + db_index=True, + help_text='The License Expression assigned to a DejaCode Product Package or Product Component is an editable value equivalent to a "concluded license" as determined by a curator who has performed analysis to clarify or correct the declared license expression, which may have been assigned automatically (from a scan or an associated package definition) when the Package or Component was originally created, or which may require the assertion of a choice of license.A license expression defines the relationship of one or more licenses to a software object. More than one applicable license can be expressed as "license-key-a AND license-key-b". A choice of applicable licenses can be expressed as "license-key-a OR license-key-b", and you can indicate the primary (preferred) license by placing it first, on the left-hand side of the OR relationship. The relationship words (OR, AND) can be combined as needed, and the use of parentheses can be applied to clarify the meaning; for example "((license-key-a AND license-key-b) OR (license-key-c))". An exception to a license can be expressed as "license-key WITH license-exception-key".', + max_length=1024, + verbose_name="Concluded license expression", + ), + ), + migrations.AlterField( + model_name="productpackage", + name="license_expression", + field=models.CharField( + blank=True, + db_index=True, + help_text='The License Expression assigned to a DejaCode Product Package or Product Component is an editable value equivalent to a "concluded license" as determined by a curator who has performed analysis to clarify or correct the declared license expression, which may have been assigned automatically (from a scan or an associated package definition) when the Package or Component was originally created, or which may require the assertion of a choice of license.A license expression defines the relationship of one or more licenses to a software object. More than one applicable license can be expressed as "license-key-a AND license-key-b". A choice of applicable licenses can be expressed as "license-key-a OR license-key-b", and you can indicate the primary (preferred) license by placing it first, on the left-hand side of the OR relationship. The relationship words (OR, AND) can be combined as needed, and the use of parentheses can be applied to clarify the meaning; for example "((license-key-a AND license-key-b) OR (license-key-c))". An exception to a license can be expressed as "license-key WITH license-exception-key".', + max_length=1024, + verbose_name="Concluded license expression", + ), + ), + ] diff --git a/product_portfolio/models.py b/product_portfolio/models.py index b14047bd..a6ffe887 100644 --- a/product_portfolio/models.py +++ b/product_portfolio/models.py @@ -42,6 +42,24 @@ from dje.validators import validate_url_segment from dje.validators import validate_version +RELATION_LICENSE_EXPRESSION_HELP_TEXT = _( + "The License Expression assigned to a DejaCode Product Package or Product " + 'Component is an editable value equivalent to a "concluded license" as determined ' + "by a curator who has performed analysis to clarify or correct the declared " + "license expression, which may have been assigned automatically " + "(from a scan or an associated package definition) when the Package or Component " + "was originally created, or which may require the assertion of a choice of license." + "A license expression defines the relationship of one or more licenses to a " + "software object. More than one applicable license can be expressed as " + '"license-key-a AND license-key-b". A choice of applicable licenses can be ' + 'expressed as "license-key-a OR license-key-b", and you can indicate the ' + "primary (preferred) license by placing it first, on the left-hand side of the " + "OR relationship. The relationship words (OR, AND) can be combined as needed, " + "and the use of parentheses can be applied to clarify the meaning; for example " + '"((license-key-a AND license-key-b) OR (license-key-c))". An exception ' + 'to a license can be expressed as "license-key WITH license-exception-key".' +) + class FieldChangesMixin: """ @@ -164,6 +182,24 @@ def get_related_secured_queryset(self, user): class Product(BaseProductMixin, FieldChangesMixin, KeywordsMixin, DataspacedModel): + license_expression = models.CharField( + max_length=1024, + blank=True, + db_index=True, + help_text=_( + "On a product in DejaCode, a license expression defines the " + "relationship of one or more licenses to that software as declared by its " + 'licensor. More than one applicable license can be expressed as "license-key-a ' + 'AND license-key-b". A choice of applicable licenses can be expressed as ' + '"license-key-a OR license-key-b", and you can indicate the primary (preferred) ' + "license by placing it first, on the left-hand side of the OR relationship. " + "The relationship words (OR, AND) can be combined as needed, and the use of " + 'parentheses can be applied to clarify the meaning; for example "((license-key-a ' + 'AND license-key-b) OR (license-key-c))". An exception to a license can be ' + 'expressed as "license-key WITH license-exception-key".' + ), + ) + is_active = models.BooleanField( verbose_name=_("active"), default=True, @@ -652,25 +688,11 @@ class ProductComponent(ProductRelationshipMixin): # it makes test_productcomponent_import_license_expression fail # This license_expression is never generated but always stored. license_expression = models.CharField( - _("License expression"), + _("Concluded license expression"), max_length=1024, blank=True, db_index=True, - help_text=_( - "On a product component relationship (which defines a component as used in your " - "product), a license expression is limited by the license(s) assigned to the original " - "component, and expresses the license(s) that apply to the context of that component " - "as it is used by your product. More than one applicable license can be expressed as " - '"license-key-a AND license-key-b". A choice of licenses can be expressed as ' - '"license-key-a OR license-key-b", and you can indicate the primary license by ' - "placing it first, on the left-hand side of the OR relationship. You can also assert " - "your license choice for the component as used in your product by editing the license " - "expression to remove any license keys that do not apply. The relationship words " - "(OR, AND) can be combined as needed, and the use of parentheses can be applied to " - 'clarify the meaning; for example "((license-key-a AND license-key-b) OR ' - '(license-key-c))". An exception to a license can be expressed as "license-key WITH ' - 'license-exception-key".' - ), + help_text=RELATION_LICENSE_EXPRESSION_HELP_TEXT, ) licenses = models.ManyToManyField( @@ -806,25 +828,11 @@ class ProductPackage(ProductRelationshipMixin): # it makes test_productcomponent_import_license_expression fail # This license_expression is never generated but always stored. license_expression = models.CharField( - _("License expression"), + _("Concluded license expression"), max_length=1024, blank=True, db_index=True, - help_text=_( - "On a product package relationship (which defines a package as used in your product), " - "a license expression is limited by the license(s) assigned to the original package " - "(unless none were assigned), and expresses the license(s) that apply to the context " - "of that package as it is used by your product. More than one applicable license can " - 'be expressed as "license-key-a AND license-key-b". A choice of licenses can be ' - 'expressed as "license-key-a OR license-key-b", and you can indicate the primary ' - "license by placing it first, on the left-hand side of the OR relationship. You can " - "also assert your license choice for the package as used in your product by editing " - "the license expression to remove any license keys that do not apply. The " - "relationship words (OR, AND) can be combined as needed, and the use of parentheses " - 'can be applied to clarify the meaning; for example "((license-key-a AND ' - 'license-key-b) OR (license-key-c))". An exception to a license can be expressed ' - 'as "license-key WITH license-exception-key".' - ), + help_text=RELATION_LICENSE_EXPRESSION_HELP_TEXT, ) licenses = models.ManyToManyField( @@ -1124,7 +1132,7 @@ class ProductInventoryItem(ProductRelationshipMixin): ) license_expression = models.CharField( - _("License expression"), + _("Concluded license expression"), max_length=1024, blank=True, ) diff --git a/product_portfolio/templates/product_portfolio/tabs/tab_inventory.html b/product_portfolio/templates/product_portfolio/tabs/tab_inventory.html index 8a34b3e8..e94fc608 100644 --- a/product_portfolio/templates/product_portfolio/tabs/tab_inventory.html +++ b/product_portfolio/templates/product_portfolio/tabs/tab_inventory.html @@ -42,7 +42,7 @@ {{ filter_productcomponent.form.purpose }} - {% trans 'License' %} + {% trans 'Concluded license' %} {% trans 'Review status' %} From 0b194be7d23a68106b098a3e85e5461914c80d43 Mon Sep 17 00:00:00 2001 From: tdruez Date: Mon, 10 Jun 2024 16:42:46 +0400 Subject: [PATCH 03/27] Add new license fields in the ComponentAdmin #63 Signed-off-by: tdruez --- component_catalog/admin.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/component_catalog/admin.py b/component_catalog/admin.py index b74a96f2..53cb0306 100644 --- a/component_catalog/admin.py +++ b/component_catalog/admin.py @@ -324,6 +324,13 @@ class ComponentAdmin( "copyright", "holder", "license_expression", + "declared_license_expression", + "declared_license_expression_spdx", + "license_detections", + "other_license_expression", + "other_license_expression_spdx", + "other_license_detections", + "extracted_license_statement", "reference_notes", "release_date", "description", @@ -418,7 +425,12 @@ class ComponentAdmin( autocomplete_lookup_fields = {"fk": ["owner"]} # We have to use 'completion_level' rather than the 'completion_level_pct' # callable to keep the help_text available during render in the template. - readonly_fields = DataspacedAdmin.readonly_fields + ("urn_link", "completion_level") + readonly_fields = DataspacedAdmin.readonly_fields + ( + "urn_link", + "completion_level", + "declared_license_expression_spdx", + "other_license_expression_spdx", + ) form = ComponentAdminForm inlines = [ SubcomponentChildInline, From 915714f1e3aec322f1f4cf3ce64be4c95e595da6 Mon Sep 17 00:00:00 2001 From: tdruez Date: Mon, 10 Jun 2024 16:58:04 +0400 Subject: [PATCH 04/27] Add new license fields in the "License" tab #63 Signed-off-by: tdruez --- dje/views.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/dje/views.py b/dje/views.py index 7d62de0a..ed3b4688 100644 --- a/dje/views.py +++ b/dje/views.py @@ -916,7 +916,15 @@ def tab_license(self): if show_usage_policy: licence_expression_source = "get_license_expression_linked_with_policy" - fields = [TabField("license_expression", source=licence_expression_source)] + fields = [ + TabField("license_expression", source=licence_expression_source), + TabField("declared_license_expression"), + TabField("declared_license_expression_spdx"), + TabField("license_detections"), + TabField("other_license_expression"), + TabField("other_license_expression_spdx"), + TabField("extracted_license_statement"), + ] if getattr(obj, "reference_notes", False): fields.append(TabField("reference_notes")) From fc697253d43c91225d3e731a2b03596417c67f89 Mon Sep 17 00:00:00 2001 From: tdruez Date: Mon, 10 Jun 2024 17:16:05 +0400 Subject: [PATCH 05/27] Add new license fields in the SCAN_PACKAGE_FIELD list #63 Signed-off-by: tdruez --- dejacode_toolkit/scancodeio.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dejacode_toolkit/scancodeio.py b/dejacode_toolkit/scancodeio.py index 332217a8..5762d073 100644 --- a/dejacode_toolkit/scancodeio.py +++ b/dejacode_toolkit/scancodeio.py @@ -256,7 +256,13 @@ def fetch_project_packages(self, project_uuid): SCAN_PACKAGE_FIELD = [ ("Package URL", "purl"), ("License expression", "declared_license_expression"), + ("Declared license expression SPDX", "declared_license_expression_spdx"), + # TODO: + # ("License detections", "license_detections"), ("Other license expression", "other_license_expression"), + ("Other license expression SPDX", "other_license_expression_spdx"), + # ("Other license detections", "other_license_detections"), + ("Extracted license statement", "extracted_license_statement"), ("Copyright", "copyright"), ("Holder", "holder"), ("Description", "description"), From 0407b1ed3c51c21b969fe057ad48b279cc8296fe Mon Sep 17 00:00:00 2001 From: tdruez Date: Fri, 14 Jun 2024 11:06:21 +0400 Subject: [PATCH 06/27] Refine the new license field on the models #131 Signed-off-by: tdruez --- component_catalog/admin.py | 16 -- component_catalog/api.py | 15 -- component_catalog/forms.py | 1 - .../migrations/0005_license_fields.py | 155 ------------------ ...ve_component_concluded_license_and_more.py | 75 +++++++++ component_catalog/models.py | 79 ++------- component_catalog/tests/test_models.py | 1 - .../component_import_byte_order_mark.csv | 2 +- component_catalog/views.py | 10 -- dejacode_toolkit/scancodeio.py | 6 - .../rest_framework/docs/description.html | 1 - dje/tests/testfiles/test_dataset_cc_only.json | 2 - dje/views.py | 2 - ...er_product_license_expression_and_more.py} | 2 +- reporting/tests/test_models.py | 1 - 15 files changed, 89 insertions(+), 279 deletions(-) delete mode 100644 component_catalog/migrations/0005_license_fields.py create mode 100644 component_catalog/migrations/0005_remove_component_concluded_license_and_more.py rename product_portfolio/migrations/{0005_license_fields.py => 0005_alter_product_license_expression_and_more.py} (98%) diff --git a/component_catalog/admin.py b/component_catalog/admin.py index 53cb0306..4b2751a4 100644 --- a/component_catalog/admin.py +++ b/component_catalog/admin.py @@ -325,12 +325,7 @@ class ComponentAdmin( "holder", "license_expression", "declared_license_expression", - "declared_license_expression_spdx", - "license_detections", "other_license_expression", - "other_license_expression_spdx", - "other_license_detections", - "extracted_license_statement", "reference_notes", "release_date", "description", @@ -401,7 +396,6 @@ class ComponentAdmin( "ip_sensitivity_approved", "affiliate_obligations", "affiliate_obligation_triggers", - "concluded_license", "legal_comments", "sublicense_allowed", "express_patent_grant", @@ -428,8 +422,6 @@ class ComponentAdmin( readonly_fields = DataspacedAdmin.readonly_fields + ( "urn_link", "completion_level", - "declared_license_expression_spdx", - "other_license_expression_spdx", ) form = ComponentAdminForm inlines = [ @@ -839,12 +831,7 @@ class PackageAdmin( "fields": ( "license_expression", "declared_license_expression", - "declared_license_expression_spdx", - "license_detections", "other_license_expression", - "other_license_expression_spdx", - "other_license_detections", - "extracted_license_statement", "copyright", "holder", "author", @@ -883,7 +870,6 @@ class PackageAdmin( "Others", { "fields": ( - "declared_license", "parties", "datasource_id", "file_references", @@ -902,8 +888,6 @@ class PackageAdmin( readonly_fields = DataspacedAdmin.readonly_fields + ( "package_url", "inferred_url", - "declared_license_expression_spdx", - "other_license_expression_spdx", ) form = PackageAdminForm importer_class = PackageImporter diff --git a/component_catalog/api.py b/component_catalog/api.py index 31cd4de7..0384cdab 100644 --- a/component_catalog/api.py +++ b/component_catalog/api.py @@ -142,11 +142,8 @@ class Meta: "license_expression", "declared_license_expression", "declared_license_expression_spdx", - "license_detections", "other_license_expression", "other_license_expression_spdx", - "other_license_detections", - "extracted_license_statement", "reference_notes", "homepage_url", "vcs_url", @@ -312,7 +309,6 @@ class Meta: "ip_sensitivity_approved", "affiliate_obligations", "affiliate_obligation_triggers", - "concluded_license", "legal_comments", "sublicense_allowed", "express_patent_grant", @@ -332,11 +328,8 @@ class Meta: "license_choices", "declared_license_expression", "declared_license_expression_spdx", - "license_detections", "other_license_expression", "other_license_expression_spdx", - "other_license_detections", - "extracted_license_statement", "created_date", "last_modified_date", ) @@ -540,11 +533,8 @@ class Meta(ComponentSerializer.Meta): "license_expression", "declared_license_expression", "declared_license_expression_spdx", - "license_detections", "other_license_expression", "other_license_expression_spdx", - "other_license_detections", - "extracted_license_statement", "reference_notes", "release_date", "description", @@ -572,7 +562,6 @@ class Meta(ComponentSerializer.Meta): "ip_sensitivity_approved", "affiliate_obligations", "affiliate_obligation_triggers", - "concluded_license", "legal_comments", "sublicense_allowed", "express_patent_grant", @@ -662,11 +651,8 @@ class Meta: "license_choices", "declared_license_expression", "declared_license_expression_spdx", - "license_detections", "other_license_expression", "other_license_expression_spdx", - "other_license_detections", - "extracted_license_statement", "reference_notes", "homepage_url", "vcs_url", @@ -684,7 +670,6 @@ class Meta: "version", "qualifiers", "subpath", - "declared_license", "parties", "datasource_id", "file_references", diff --git a/component_catalog/forms.py b/component_catalog/forms.py index e6c3ddf8..92b58596 100644 --- a/component_catalog/forms.py +++ b/component_catalog/forms.py @@ -1043,7 +1043,6 @@ class Meta: "ip_sensitivity_approved", "affiliate_obligations", "affiliate_obligation_triggers", - "concluded_license", "legal_comments", "sublicense_allowed", "express_patent_grant", diff --git a/component_catalog/migrations/0005_license_fields.py b/component_catalog/migrations/0005_license_fields.py deleted file mode 100644 index d448a8e3..00000000 --- a/component_catalog/migrations/0005_license_fields.py +++ /dev/null @@ -1,155 +0,0 @@ -# Generated by Django 5.0.6 on 2024-06-10 06:37 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("component_catalog", "0004_initial"), - ] - - operations = [ - migrations.AddField( - model_name="component", - name="declared_license_expression", - field=models.TextField( - blank=True, - help_text="A license expression derived from statements in the key files of a software project, such as the NOTICE, COPYING, README, and LICENSE files.", - ), - ), - migrations.AddField( - model_name="component", - name="declared_license_expression_spdx", - field=models.TextField( - blank=True, - help_text="A declared license expression that uses the license identifiers defined by SPDX, as well as the “Licenseref” syntax for licenses not on the SPDX list.", - verbose_name="Declared license expression SPDX", - ), - ), - migrations.AddField( - model_name="component", - name="extracted_license_statement", - field=models.TextField( - blank=True, - help_text="The actual text extracted from a software project that supports the license detection process.", - ), - ), - migrations.AddField( - model_name="component", - name="license_detections", - field=models.JSONField( - blank=True, - default=list, - help_text="A list of specific license identifiers derived from statements in the key files of a software project, such as the NOTICE, COPYING, README, and LICENSE files.", - ), - ), - migrations.AddField( - model_name="component", - name="other_license_detections", - field=models.JSONField( - blank=True, - default=list, - help_text="A list of specific license identifiers derived from the non-key files of a software project, which are very often third-party software used by the project, or test, sample and documentation files.", - ), - ), - migrations.AddField( - model_name="component", - name="other_license_expression", - field=models.TextField( - blank=True, - help_text="A license expression derived from detected licenses in the non-key files of a software project, which are often third-party software used by the project, or test, sample and documentation files.", - ), - ), - migrations.AddField( - model_name="component", - name="other_license_expression_spdx", - field=models.TextField( - blank=True, - help_text="A license expression derived from detected licenses in the non-key files of a software project, using the license identifiers defined by SPDX, as well as the “Licenseref” syntax for licenses not on the SPDX list.", - verbose_name="Other license expression SPDX", - ), - ), - migrations.AddField( - model_name="package", - name="declared_license_expression", - field=models.TextField( - blank=True, - help_text="A license expression derived from statements in the key files of a software project, such as the NOTICE, COPYING, README, and LICENSE files.", - ), - ), - migrations.AddField( - model_name="package", - name="declared_license_expression_spdx", - field=models.TextField( - blank=True, - help_text="A declared license expression that uses the license identifiers defined by SPDX, as well as the “Licenseref” syntax for licenses not on the SPDX list.", - verbose_name="Declared license expression SPDX", - ), - ), - migrations.AddField( - model_name="package", - name="extracted_license_statement", - field=models.TextField( - blank=True, - help_text="The actual text extracted from a software project that supports the license detection process.", - ), - ), - migrations.AddField( - model_name="package", - name="license_detections", - field=models.JSONField( - blank=True, - default=list, - help_text="A list of specific license identifiers derived from statements in the key files of a software project, such as the NOTICE, COPYING, README, and LICENSE files.", - ), - ), - migrations.AddField( - model_name="package", - name="other_license_detections", - field=models.JSONField( - blank=True, - default=list, - help_text="A list of specific license identifiers derived from the non-key files of a software project, which are very often third-party software used by the project, or test, sample and documentation files.", - ), - ), - migrations.AddField( - model_name="package", - name="other_license_expression", - field=models.TextField( - blank=True, - help_text="A license expression derived from detected licenses in the non-key files of a software project, which are often third-party software used by the project, or test, sample and documentation files.", - ), - ), - migrations.AddField( - model_name="package", - name="other_license_expression_spdx", - field=models.TextField( - blank=True, - help_text="A license expression derived from detected licenses in the non-key files of a software project, using the license identifiers defined by SPDX, as well as the “Licenseref” syntax for licenses not on the SPDX list.", - verbose_name="Other license expression SPDX", - ), - ), - migrations.AlterField( - model_name="component", - name="license_expression", - field=models.CharField( - blank=True, - db_index=True, - help_text='The License Expression assigned to a DejaCode Package or Component is an editable value equivalent to a "concluded license" as determined by a curator who has performed analysis to clarify or correct the declared license expression, which may have been assigned automatically (from a scan or an associated package definition) when the Package or Component was originally created. A license expression defines the relationship of one or more licenses to a software object. More than one applicable license can be expressed as "license-key-a AND license-key-b". A choice of applicable licenses can be expressed as "license-key-a OR license-key-b", and you can indicate the primary (preferred) license by placing it first, on the left-hand side of the OR relationship. The relationship words (OR, AND) can be combined as needed, and the use of parentheses can be applied to clarify the meaning; for example "((license-key-a AND license-key-b) OR (license-key-c))". An exception to a license can be expressed as "license-key WITH license-exception-key".', - max_length=1024, - verbose_name="Concluded license expression", - ), - ), - migrations.AlterField( - model_name="package", - name="license_expression", - field=models.CharField( - blank=True, - db_index=True, - help_text='The License Expression assigned to a DejaCode Package or Component is an editable value equivalent to a "concluded license" as determined by a curator who has performed analysis to clarify or correct the declared license expression, which may have been assigned automatically (from a scan or an associated package definition) when the Package or Component was originally created. A license expression defines the relationship of one or more licenses to a software object. More than one applicable license can be expressed as "license-key-a AND license-key-b". A choice of applicable licenses can be expressed as "license-key-a OR license-key-b", and you can indicate the primary (preferred) license by placing it first, on the left-hand side of the OR relationship. The relationship words (OR, AND) can be combined as needed, and the use of parentheses can be applied to clarify the meaning; for example "((license-key-a AND license-key-b) OR (license-key-c))". An exception to a license can be expressed as "license-key WITH license-exception-key".', - max_length=1024, - verbose_name="Concluded license expression", - ), - ), - ] diff --git a/component_catalog/migrations/0005_remove_component_concluded_license_and_more.py b/component_catalog/migrations/0005_remove_component_concluded_license_and_more.py new file mode 100644 index 00000000..e332e569 --- /dev/null +++ b/component_catalog/migrations/0005_remove_component_concluded_license_and_more.py @@ -0,0 +1,75 @@ +# Generated by Django 5.0.6 on 2024-06-14 07:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("component_catalog", "0004_initial"), + ] + + operations = [ + migrations.RemoveField( + model_name="component", + name="concluded_license", + ), + migrations.RemoveField( + model_name="package", + name="declared_license", + ), + migrations.AddField( + model_name="component", + name="declared_license_expression", + field=models.TextField( + blank=True, + help_text="A license expression derived from statements in the manifests or key files of a software project, such as the NOTICE, COPYING, README, and LICENSE files.", + ), + ), + migrations.AddField( + model_name="component", + name="other_license_expression", + field=models.TextField( + blank=True, + help_text="A license expression derived from detected licenses in the non-key files of a software project, which are often third-party software used by the project, or test, sample and documentation files.", + ), + ), + migrations.AddField( + model_name="package", + name="declared_license_expression", + field=models.TextField( + blank=True, + help_text="A license expression derived from statements in the manifests or key files of a software project, such as the NOTICE, COPYING, README, and LICENSE files.", + ), + ), + migrations.AddField( + model_name="package", + name="other_license_expression", + field=models.TextField( + blank=True, + help_text="A license expression derived from detected licenses in the non-key files of a software project, which are often third-party software used by the project, or test, sample and documentation files.", + ), + ), + migrations.AlterField( + model_name="component", + name="license_expression", + field=models.CharField( + blank=True, + db_index=True, + help_text='The License Expression assigned to a DejaCode Package or Component is an editable value equivalent to a "concluded license" as determined by a curator who has performed analysis to clarify or correct the declared license expression, which may have been assigned automatically (from a scan or an associated package definition) when the Package or Component was originally created. A license expression defines the relationship of one or more licenses to a software object. More than one applicable license can be expressed as "license-key-a AND license-key-b". A choice of applicable licenses can be expressed as "license-key-a OR license-key-b", and you can indicate the primary (preferred) license by placing it first, on the left-hand side of the OR relationship. The relationship words (OR, AND) can be combined as needed, and the use of parentheses can be applied to clarify the meaning; for example "((license-key-a AND license-key-b) OR (license-key-c))". An exception to a license can be expressed as "license-key WITH license-exception-key".', + max_length=1024, + verbose_name="Concluded license expression", + ), + ), + migrations.AlterField( + model_name="package", + name="license_expression", + field=models.CharField( + blank=True, + db_index=True, + help_text='The License Expression assigned to a DejaCode Package or Component is an editable value equivalent to a "concluded license" as determined by a curator who has performed analysis to clarify or correct the declared license expression, which may have been assigned automatically (from a scan or an associated package definition) when the Package or Component was originally created. A license expression defines the relationship of one or more licenses to a software object. More than one applicable license can be expressed as "license-key-a AND license-key-b". A choice of applicable licenses can be expressed as "license-key-a OR license-key-b", and you can indicate the primary (preferred) license by placing it first, on the left-hand side of the OR relationship. The relationship words (OR, AND) can be combined as needed, and the use of parentheses can be applied to clarify the meaning; for example "((license-key-a AND license-key-b) OR (license-key-c))". An exception to a license can be expressed as "license-key WITH license-exception-key".', + max_length=1024, + verbose_name="Concluded license expression", + ), + ), + ] diff --git a/component_catalog/models.py b/component_catalog/models.py index 47554651..0d407464 100644 --- a/component_catalog/models.py +++ b/component_catalog/models.py @@ -323,28 +323,12 @@ class LicenseFieldsMixin(models.Model): declared_license_expression = models.TextField( blank=True, help_text=_( - "A license expression derived from statements in the key files of a " - "software project, such as the NOTICE, COPYING, README, and LICENSE files." - ), - ) - declared_license_expression_spdx = models.TextField( - verbose_name=_("Declared license expression SPDX"), - blank=True, - help_text=_( - "A declared license expression that uses the license identifiers defined " - "by SPDX, as well as the “Licenseref” syntax for licenses not on the SPDX " - "list." - ), - ) - license_detections = models.JSONField( - default=list, - blank=True, - help_text=_( - "A list of specific license identifiers derived from statements in the " - "key files of a software project, such as the NOTICE, COPYING, README, " - "and LICENSE files." + "A license expression derived from statements in the manifests or key " + "files of a software project, such as the NOTICE, COPYING, README, and " + "LICENSE files." ), ) + other_license_expression = models.TextField( blank=True, help_text=_( @@ -353,35 +337,18 @@ class LicenseFieldsMixin(models.Model): "project, or test, sample and documentation files." ), ) - other_license_expression_spdx = models.TextField( - verbose_name=_("Other license expression SPDX"), - blank=True, - help_text=_( - "A license expression derived from detected licenses in the non-key files " - "of a software project, using the license identifiers defined by SPDX, " - "as well as the “Licenseref” syntax for licenses not on the SPDX list." - ), - ) - other_license_detections = models.JSONField( - default=list, - blank=True, - help_text=_( - "A list of specific license identifiers derived from the non-key files " - "of a software project, which are very often third-party software used " - "by the project, or test, sample and documentation files." - ), - ) - extracted_license_statement = models.TextField( - blank=True, - help_text=_( - "The actual text extracted from a software project that supports the " - "license detection process." - ), - ) class Meta: abstract = True + @property + def declared_license_expression_spdx(self): + return "TODO" + + @property + def other_license_expression_spdx(self): + return "TODO" + def get_cyclonedx_properties(instance): """ @@ -1099,19 +1066,6 @@ class Component( ), ) - concluded_license = models.CharField( - max_length=1024, - blank=True, - db_index=True, - help_text=_( - "This is a memo field to record the conclusions of the legal team after full review " - "and scanning of the component package, and is only intended to document that " - "decision. The main value of the field is to clarify the company interpretation of " - "the license that should apply to a component when there is a choice, or when there " - "is ambiguity in the original component documentation." - ), - ) - legal_comments = models.TextField( blank=True, help_text=_( @@ -1823,15 +1777,6 @@ class Package( ), ) - declared_license = models.TextField( - blank=True, - help_text=_( - "The declared license mention, tag or text as found in a package manifest. " - "This can be a string, a list or dict of strings possibly nested, " - "as found originally in the manifest." - ), - ) - datasource_id = models.CharField( max_length=64, blank=True, diff --git a/component_catalog/tests/test_models.py b/component_catalog/tests/test_models.py index dfe2a10d..9897d42b 100644 --- a/component_catalog/tests/test_models.py +++ b/component_catalog/tests/test_models.py @@ -1254,7 +1254,6 @@ def test_component_catalog_models_get_exclude_candidates_fields(self): "ip_sensitivity_approved", "affiliate_obligations", "affiliate_obligation_triggers", - "concluded_license", "keywords", "legal_comments", "sublicense_allowed", diff --git a/component_catalog/tests/testfiles/import/component_import_byte_order_mark.csv b/component_catalog/tests/testfiles/import/component_import_byte_order_mark.csv index f4081516..f2bf0359 100644 --- a/component_catalog/tests/testfiles/import/component_import_byte_order_mark.csv +++ b/component_catalog/tests/testfiles/import/component_import_byte_order_mark.csv @@ -1,2 +1,2 @@ -name,reference_notes,usage_policy,request_count,version,owner,release_date,description,copyright,homepage_url,vcs_url,code_view_url,bug_tracking_url,primary_language,admin_notes,notice_text,license_expression,configuration_status,type,approval_reference,guidance,is_active,curation_level,is_license_notice,is_copyright_notice,is_notice_in_codebase,notice_filename,notice_url,dependencies,project,codescan_identifier,website_terms_of_use,ip_sensitivity_approved,affiliate_obligations,affiliate_obligation_triggers,concluded_license,legal_comments,sublicense_allowed,legal_reviewed,distribution_formats_allowed,acceptable_linkages,export_restrictions,approved_download_location,approved_community_interaction,keyword +name,reference_notes,usage_policy,request_count,version,owner,release_date,description,copyright,homepage_url,vcs_url,code_view_url,bug_tracking_url,primary_language,admin_notes,notice_text,license_expression,configuration_status,type,approval_reference,guidance,is_active,curation_level,is_license_notice,is_copyright_notice,is_notice_in_codebase,notice_filename,notice_url,dependencies,project,codescan_identifier,website_terms_of_use,ip_sensitivity_approved,affiliate_obligations,affiliate_obligation_triggers,legal_comments,sublicense_allowed,legal_reviewed,distribution_formats_allowed,acceptable_linkages,export_restrictions,approved_download_location,approved_community_interaction,keyword Partita,test ref notes,,,5.1.3,Unspecified,4/1/19,test desc,Copyright somebody,https://en.wikipedia.org/wiki/Blaise_Pascal,,,,Pascal,,test notice text,mit AND other-permissive,,,,,Yes,35,Yes,Yes,Yes,test.txt,https://en.wikipedia.org/wiki/Blaise_Pascal,,test,,,,,,,,,,,,,,,Framework \ No newline at end of file diff --git a/component_catalog/views.py b/component_catalog/views.py index ea9d48fa..1d3bbce9 100644 --- a/component_catalog/views.py +++ b/component_catalog/views.py @@ -563,11 +563,8 @@ class ComponentDetailsView( "licenses_summary", "declared_license_expression", "declared_license_expression_spdx", - "license_detections", "other_license_expression", "other_license_expression_spdx", - "other_license_detections", - "extracted_license_statement", ], }, "hierarchy": {}, @@ -604,7 +601,6 @@ class ComponentDetailsView( "ip_sensitivity_approved", "affiliate_obligations", "affiliate_obligation_triggers", - "concluded_license", "legal_comments", "sublicense_allowed", "express_patent_grant", @@ -866,7 +862,6 @@ def tab_legal(self): TabField("ip_sensitivity_approved"), TabField("affiliate_obligations"), TabField("affiliate_obligation_triggers"), - TabField("concluded_license"), TabField("legal_comments"), TabField("sublicense_allowed"), TabField("express_patent_grant"), @@ -1222,11 +1217,8 @@ class PackageDetailsView( "licenses_summary", "declared_license_expression", "declared_license_expression_spdx", - "license_detections", "other_license_expression", "other_license_expression_spdx", - "other_license_detections", - "extracted_license_statement", ], }, "terms": { @@ -1260,7 +1252,6 @@ class PackageDetailsView( }, "others": { "fields": [ - "declared_license", "parties", "datasource_id", "file_references", @@ -1413,7 +1404,6 @@ def tab_checksums(self): def tab_others(self): tab_fields = [ - TabField("declared_license"), TabField("parties"), TabField("datasource_id"), TabField("file_references"), diff --git a/dejacode_toolkit/scancodeio.py b/dejacode_toolkit/scancodeio.py index 5762d073..332217a8 100644 --- a/dejacode_toolkit/scancodeio.py +++ b/dejacode_toolkit/scancodeio.py @@ -256,13 +256,7 @@ def fetch_project_packages(self, project_uuid): SCAN_PACKAGE_FIELD = [ ("Package URL", "purl"), ("License expression", "declared_license_expression"), - ("Declared license expression SPDX", "declared_license_expression_spdx"), - # TODO: - # ("License detections", "license_detections"), ("Other license expression", "other_license_expression"), - ("Other license expression SPDX", "other_license_expression_spdx"), - # ("Other license detections", "other_license_detections"), - ("Extracted license statement", "extracted_license_statement"), ("Copyright", "copyright"), ("Holder", "holder"), ("Description", "description"), diff --git a/dje/templates/rest_framework/docs/description.html b/dje/templates/rest_framework/docs/description.html index b2171a1d..51bdcaa9 100644 --- a/dje/templates/rest_framework/docs/description.html +++ b/dje/templates/rest_framework/docs/description.html @@ -313,7 +313,6 @@

Create a Component

"ip_sensitivity_approved": false, "affiliate_obligations": false, "affiliate_obligation_triggers": "", - "concluded_license": "", "legal_comments": "", "sublicense_allowed": null, "express_patent_grant": null, diff --git a/dje/tests/testfiles/test_dataset_cc_only.json b/dje/tests/testfiles/test_dataset_cc_only.json index 35d19f16..c209c233 100644 --- a/dje/tests/testfiles/test_dataset_cc_only.json +++ b/dje/tests/testfiles/test_dataset_cc_only.json @@ -88,7 +88,6 @@ "ip_sensitivity_approved": false, "affiliate_obligations": false, "affiliate_obligation_triggers": "", - "concluded_license": "", "legal_comments": "", "sublicense_allowed": null, "express_patent_grant": null, @@ -157,7 +156,6 @@ "ip_sensitivity_approved": false, "affiliate_obligations": false, "affiliate_obligation_triggers": "", - "concluded_license": "", "legal_comments": "", "sublicense_allowed": null, "express_patent_grant": null, diff --git a/dje/views.py b/dje/views.py index ed3b4688..801f5cce 100644 --- a/dje/views.py +++ b/dje/views.py @@ -920,10 +920,8 @@ def tab_license(self): TabField("license_expression", source=licence_expression_source), TabField("declared_license_expression"), TabField("declared_license_expression_spdx"), - TabField("license_detections"), TabField("other_license_expression"), TabField("other_license_expression_spdx"), - TabField("extracted_license_statement"), ] if getattr(obj, "reference_notes", False): diff --git a/product_portfolio/migrations/0005_license_fields.py b/product_portfolio/migrations/0005_alter_product_license_expression_and_more.py similarity index 98% rename from product_portfolio/migrations/0005_license_fields.py rename to product_portfolio/migrations/0005_alter_product_license_expression_and_more.py index 9bb0b528..69edc062 100644 --- a/product_portfolio/migrations/0005_license_fields.py +++ b/product_portfolio/migrations/0005_alter_product_license_expression_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 5.0.6 on 2024-06-10 06:37 +# Generated by Django 5.0.6 on 2024-06-14 07:05 from django.db import migrations, models diff --git a/reporting/tests/test_models.py b/reporting/tests/test_models.py index 0afbd218..a1dac023 100644 --- a/reporting/tests/test_models.py +++ b/reporting/tests/test_models.py @@ -1928,7 +1928,6 @@ def test_get_model_data_for_component_column_template(self): "value": "codescan_identifier", }, {"group": "Direct Fields", "label": "completion_level", "value": "completion_level"}, - {"group": "Direct Fields", "label": "concluded_license", "value": "concluded_license"}, { "group": "Direct Fields", "label": "configuration_status >>", From e1b47b1bf6317d46e791c9903e75f0c36dbde3fe Mon Sep 17 00:00:00 2001 From: tdruez Date: Fri, 14 Jun 2024 11:39:01 +0400 Subject: [PATCH 07/27] Refine the main Component and Package forms #131 Signed-off-by: tdruez --- component_catalog/api.py | 8 -------- component_catalog/forms.py | 10 ++++++++++ component_catalog/views.py | 4 ---- dje/views.py | 2 -- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/component_catalog/api.py b/component_catalog/api.py index 0384cdab..c7064531 100644 --- a/component_catalog/api.py +++ b/component_catalog/api.py @@ -141,9 +141,7 @@ class Meta: "author", "license_expression", "declared_license_expression", - "declared_license_expression_spdx", "other_license_expression", - "other_license_expression_spdx", "reference_notes", "homepage_url", "vcs_url", @@ -327,9 +325,7 @@ class Meta: "license_choices_expression", "license_choices", "declared_license_expression", - "declared_license_expression_spdx", "other_license_expression", - "other_license_expression_spdx", "created_date", "last_modified_date", ) @@ -532,9 +528,7 @@ class Meta(ComponentSerializer.Meta): "holder", "license_expression", "declared_license_expression", - "declared_license_expression_spdx", "other_license_expression", - "other_license_expression_spdx", "reference_notes", "release_date", "description", @@ -650,9 +644,7 @@ class Meta: "license_choices_expression", "license_choices", "declared_license_expression", - "declared_license_expression_spdx", "other_license_expression", - "other_license_expression_spdx", "reference_notes", "homepage_url", "vcs_url", diff --git a/component_catalog/forms.py b/component_catalog/forms.py index 92b58596..b41329de 100644 --- a/component_catalog/forms.py +++ b/component_catalog/forms.py @@ -106,6 +106,8 @@ class Meta: "holder", "notice_text", "license_expression", + "declared_license_expression", + "other_license_expression", "release_date", "description", "homepage_url", @@ -130,6 +132,8 @@ class Meta: "owner": OwnerChoiceField, } widgets = { + "declared_license_expression": forms.Textarea(attrs={"rows": 2}), + "other_license_expression": forms.Textarea(attrs={"rows": 2}), "copyright": forms.Textarea(attrs={"rows": 2}), "notice_text": forms.Textarea(attrs={"rows": 2}), "description": forms.Textarea(attrs={"rows": 2}), @@ -188,6 +192,7 @@ def helper(self): Group("name", "version", "owner"), HTML("
"), "license_expression", + Group("declared_license_expression", "other_license_expression"), Group("copyright", "holder"), "notice_text", Group("notice_filename", "notice_url"), @@ -299,6 +304,8 @@ class Meta: "notes", "usage_policy", "license_expression", + "declared_license_expression", + "other_license_expression", "copyright", "holder", "author", @@ -320,6 +327,8 @@ class Meta: "collect_data", ] widgets = { + "declared_license_expression": forms.Textarea(attrs={"rows": 2}), + "other_license_expression": forms.Textarea(attrs={"rows": 2}), "description": forms.Textarea(attrs={"rows": 2}), "notes": forms.Textarea(attrs={"rows": 2}), "copyright": forms.Textarea(attrs={"rows": 2}), @@ -377,6 +386,7 @@ def helper(self): Group("version", "qualifiers", "subpath"), HTML("
"), "license_expression", + Group("declared_license_expression", "other_license_expression"), Group("copyright", "notice_text"), Group("holder", "author"), HTML("
"), diff --git a/component_catalog/views.py b/component_catalog/views.py index 1d3bbce9..9f59db65 100644 --- a/component_catalog/views.py +++ b/component_catalog/views.py @@ -562,9 +562,7 @@ class ComponentDetailsView( "licenses", "licenses_summary", "declared_license_expression", - "declared_license_expression_spdx", "other_license_expression", - "other_license_expression_spdx", ], }, "hierarchy": {}, @@ -1216,9 +1214,7 @@ class PackageDetailsView( "licenses", "licenses_summary", "declared_license_expression", - "declared_license_expression_spdx", "other_license_expression", - "other_license_expression_spdx", ], }, "terms": { diff --git a/dje/views.py b/dje/views.py index 801f5cce..7171d0b9 100644 --- a/dje/views.py +++ b/dje/views.py @@ -919,9 +919,7 @@ def tab_license(self): fields = [ TabField("license_expression", source=licence_expression_source), TabField("declared_license_expression"), - TabField("declared_license_expression_spdx"), TabField("other_license_expression"), - TabField("other_license_expression_spdx"), ] if getattr(obj, "reference_notes", False): From a8259f6daa1ab29830ec0911bff94077a0740cdb Mon Sep 17 00:00:00 2001 From: tdruez Date: Fri, 14 Jun 2024 17:06:11 +0400 Subject: [PATCH 08/27] Add support for new fields in the DETECTED PACKAGE Scan tab section #131 Signed-off-by: tdruez --- component_catalog/forms.py | 14 ++++++++++++-- component_catalog/views.py | 5 +---- dejacode_toolkit/scancodeio.py | 22 ++++++++++------------ dje/views.py | 15 ++++++++------- 4 files changed, 31 insertions(+), 25 deletions(-) diff --git a/component_catalog/forms.py b/component_catalog/forms.py index b41329de..b7796dd0 100644 --- a/component_catalog/forms.py +++ b/component_catalog/forms.py @@ -14,6 +14,7 @@ from django.urls import reverse from django.urls import reverse_lazy from django.utils.functional import cached_property +from django.utils.text import Truncator import packageurl from crispy_forms.helper import FormHelper @@ -450,6 +451,7 @@ def helper(self): class ScanToPackageForm(BaseScanToPackageForm): prefix = "scan-to-package" + expression_field_names = ["declared_license_expression", "other_license_expression"] package_url = forms.CharField( label="Package URL", @@ -461,6 +463,8 @@ class Meta: fields = [ "package_url", "license_expression", + "declared_license_expression", + "other_license_expression", "copyright", "primary_language", "description", @@ -492,6 +496,12 @@ def __init__(self, *args, **kwargs): def fields_with_initial_value(self): kept_fields = {} + # Duplicate the declared_license_expression into the license_expression field + # if currently empty on the Package instance. + if not self.instance.license_expression: + if declared_license_expression := self.initial.get("declared_license_expression"): + self.initial["license_expression"] = declared_license_expression + for field_name, field in self.fields.items(): if not self.initial.get(field_name): continue @@ -499,7 +509,7 @@ def fields_with_initial_value(self): instance_value = getattr(self.instance, field_name, None) help_text = "No current value" if instance_value: - help_text = f"Current value: {instance_value}" + help_text = f"Current value: {Truncator(instance_value).chars(200)}" field.help_text = help_text kept_fields[field_name] = field @@ -552,7 +562,7 @@ def set_help_text_with_initial_value(self): instance_value = getattr(self.instance, field_name, None) help_text = "No current value" if instance_value: - help_text = f"Current value: {instance_value}" + help_text = f"Current value: {Truncator(instance_value).chars(200)}" field.help_text = help_text diff --git a/component_catalog/views.py b/component_catalog/views.py index 9f59db65..7a037a54 100644 --- a/component_catalog/views.py +++ b/component_catalog/views.py @@ -2307,10 +2307,7 @@ def scan_detected_package_fields(self, key_files_packages): # Add the supported Package fields in the tab UI. for label, scan_field in ScanCodeIO.SCAN_PACKAGE_FIELD: - if scan_field == "declared_license_expression": - scan_field = "license_expression" - value = self.detected_package_data.get(scan_field) - if value: + if value := self.detected_package_data.get(scan_field): if isinstance(value, list): value = format_html("
".join(([escape(entry) for entry in value]))) else: diff --git a/dejacode_toolkit/scancodeio.py b/dejacode_toolkit/scancodeio.py index 332217a8..728dda88 100644 --- a/dejacode_toolkit/scancodeio.py +++ b/dejacode_toolkit/scancodeio.py @@ -255,8 +255,8 @@ def fetch_project_packages(self, project_uuid): # (label, scan_field) SCAN_PACKAGE_FIELD = [ ("Package URL", "purl"), - ("License expression", "declared_license_expression"), - ("Other license expression", "other_license_expression"), + ("Declared license", "declared_license_expression"), + ("Other license", "other_license_expression"), ("Copyright", "copyright"), ("Holder", "holder"), ("Description", "description"), @@ -354,6 +354,8 @@ def fetch_project_packages(self, project_uuid): # (scan_field, model_field) AUTO_UPDATE_FIELDS = [ ("declared_license_expression", "license_expression"), + ("declared_license_expression", "declared_license_expression"), + ("other_license_expression", "other_license_expression"), ("declared_holder", "holder"), ("primary_language", "primary_language"), ] @@ -366,26 +368,22 @@ def map_detected_package_data(cls, detected_package): """ package_data_for_model = {} - for _, scan_field in cls.SCAN_PACKAGE_FIELD: - value = detected_package.get(scan_field) + for _, scan_data_field in cls.SCAN_PACKAGE_FIELD: + value = detected_package.get(scan_data_field) if not value: continue - if scan_field == "dependencies": + if scan_data_field == "dependencies": value = json.dumps(value, indent=2) - if scan_field == "other_license_expression": - continue - # Add `package_url` alias to be used in place of `purl` depending on the context - if scan_field == "purl": + if scan_data_field == "purl": package_data_for_model["package_url"] = value - elif scan_field == "declared_license_expression": - scan_field = "license_expression" + elif scan_data_field.endswith("license_expression"): value = str(Licensing().dedup(value)) - package_data_for_model[scan_field] = value + package_data_for_model[scan_data_field] = value return package_data_for_model diff --git a/dje/views.py b/dje/views.py index 7171d0b9..1e066aab 100644 --- a/dje/views.py +++ b/dje/views.py @@ -906,24 +906,25 @@ def tab_license(self): """Return a mapping of data for use in the license tab display or None.""" obj = self.object licenses = get_license_objects(obj.license_expression, obj.licensing) - - if not licenses: - return - show_usage_policy = self.request.user.dataspace.show_usage_policy_in_user_views licence_expression_source = "license_expression_linked" if show_usage_policy: licence_expression_source = "get_license_expression_linked_with_policy" - fields = [ + tab_fields = [ TabField("license_expression", source=licence_expression_source), TabField("declared_license_expression"), TabField("other_license_expression"), ] if getattr(obj, "reference_notes", False): - fields.append(TabField("reference_notes")) + tab_fields.append(TabField("reference_notes")) + + fields = self.get_tab_fields(tab_fields) + # At least 1 value need to be set for the tab to be available. + if not any([1 for field in fields if field[1]]): + return license_conditions_help = _( "A list of all the license conditions (obligations, restrictions, policies) that " @@ -949,7 +950,7 @@ def tab_license(self): } return { - "fields": self.get_tab_fields(fields), + "fields": fields, "extra": extra, } From 37cc0ba2377cdc41771460a3369710c3d47dffed Mon Sep 17 00:00:00 2001 From: tdruez Date: Fri, 14 Jun 2024 17:24:22 +0400 Subject: [PATCH 09/27] WIP on the scan summary support #131 Signed-off-by: tdruez --- component_catalog/forms.py | 9 ++++++++- .../includes/scan_summary_to_package_modal.html | 9 +++++---- dejacode_toolkit/scancodeio.py | 9 +++++++-- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/component_catalog/forms.py b/component_catalog/forms.py index b7796dd0..fbb30231 100644 --- a/component_catalog/forms.py +++ b/component_catalog/forms.py @@ -440,6 +440,12 @@ def save(self, *args, **kwargs): class BaseScanToPackageForm(LicenseExpressionFormMixin, DataspacedModelForm): + expression_field_names = [ + "license_expression", + "declared_license_expression", + "other_license_expression", + ] + @property def helper(self): helper = FormHelper() @@ -451,7 +457,6 @@ def helper(self): class ScanToPackageForm(BaseScanToPackageForm): prefix = "scan-to-package" - expression_field_names = ["declared_license_expression", "other_license_expression"] package_url = forms.CharField( label="Package URL", @@ -544,6 +549,8 @@ class Meta: model = Package fields = [ "license_expression", + "declared_license_expression", + "other_license_expression", "primary_language", "holder", ] diff --git a/component_catalog/templates/component_catalog/includes/scan_summary_to_package_modal.html b/component_catalog/templates/component_catalog/includes/scan_summary_to_package_modal.html index e6af3aa6..8af7a492 100644 --- a/component_catalog/templates/component_catalog/includes/scan_summary_to_package_modal.html +++ b/component_catalog/templates/component_catalog/includes/scan_summary_to_package_modal.html @@ -20,6 +20,7 @@
Set values from Scan Summary to Package
\ No newline at end of file diff --git a/component_catalog/templates/component_catalog/tabs/tab_scan.html b/component_catalog/templates/component_catalog/tabs/tab_scan.html index a19f63dc..200fda71 100644 --- a/component_catalog/templates/component_catalog/tabs/tab_scan.html +++ b/component_catalog/templates/component_catalog/tabs/tab_scan.html @@ -17,7 +17,6 @@ NEXB.displayOverlay("Submitting Scan Request..."); }); - {% else %} {% include 'tabs/tab_content.html' %} {% if object.has_license_matches %} diff --git a/component_catalog/tests/test_views.py b/component_catalog/tests/test_views.py index da23b690..8527e79a 100644 --- a/component_catalog/tests/test_views.py +++ b/component_catalog/tests/test_views.py @@ -2818,7 +2818,7 @@ def test_package_details_view_get_license_expressions_scan_values(self): } values = PackageTabScanView.get_license_expressions_scan_values( - self.dataspace, field_data, input_type, license_matches + self.dataspace, field_data, "license_expression", input_type, license_matches ) self.assertEqual(1, len(values)) self.assertIn("MATCH", values[0]) diff --git a/component_catalog/views.py b/component_catalog/views.py index 7a037a54..4467024a 100644 --- a/component_catalog/views.py +++ b/component_catalog/views.py @@ -2081,7 +2081,13 @@ def _get_licensing_for_formatted_render(dataspace, license_expressions): @classmethod def get_license_expressions_scan_values( - cls, dataspace, license_expressions, input_type, license_matches, checked=False + cls, + dataspace, + license_expressions, + field_name, + input_type, + license_matches, + checked=False, ): licensing, show_policy = cls._get_licensing_for_formatted_render( dataspace, license_expressions @@ -2096,7 +2102,7 @@ def get_license_expressions_scan_values( count = entry.get("count") checked_html = "checked" if checked else "" select_input = ( - f'' ) @@ -2347,7 +2353,6 @@ def scan_summary_fields(self, scan_summary): ) license_matches = scan_summary.get("license_matches") or {} self.object.has_license_matches = bool(license_matches) - for label, field, model_field_name, input_type in summary_fields: field_data = scan_summary.get(field, []) if field in ("declared_license_expression", "other_license_expressions"): @@ -2357,7 +2362,12 @@ def scan_summary_fields(self, scan_summary): else: checked = False values = self.get_license_expressions_scan_values( - user.dataspace, field_data, input_type, license_matches, checked + user.dataspace, + field_data, + model_field_name, + input_type, + license_matches, + checked, ) elif field in ("declared_holder", "primary_language"): diff --git a/dejacode/static/css/dejacode_bootstrap.css b/dejacode/static/css/dejacode_bootstrap.css index 0365875d..07d374cc 100644 --- a/dejacode/static/css/dejacode_bootstrap.css +++ b/dejacode/static/css/dejacode_bootstrap.css @@ -494,6 +494,9 @@ div.awesomplete { max-height: 50vh; overflow-y: auto; } +.awesomplete > ul:before { + display: none; +} [data-bs-theme=dark] .awesomplete > ul { background: var(--bs-black); } From 14e715f6d1244e67fa765b79764bfae87f43b8ac Mon Sep 17 00:00:00 2001 From: tdruez Date: Wed, 19 Jun 2024 17:04:07 +0400 Subject: [PATCH 11/27] Add autocomplete for all license_expression fields in "Set values to Package" #63 Signed-off-by: tdruez --- .../includes/scan_matches_modal.html | 16 +++++++--------- .../includes/scan_summary_to_package_modal.html | 4 ++-- .../includes/scan_to_package_modal.html | 15 ++++++++++++--- dje/views.py | 6 ++++-- 4 files changed, 25 insertions(+), 16 deletions(-) diff --git a/component_catalog/templates/component_catalog/includes/scan_matches_modal.html b/component_catalog/templates/component_catalog/includes/scan_matches_modal.html index e94b2365..79b2c860 100644 --- a/component_catalog/templates/component_catalog/includes/scan_matches_modal.html +++ b/component_catalog/templates/component_catalog/includes/scan_matches_modal.html @@ -17,14 +17,12 @@
\ No newline at end of file diff --git a/component_catalog/templates/component_catalog/includes/scan_summary_to_package_modal.html b/component_catalog/templates/component_catalog/includes/scan_summary_to_package_modal.html index eef563f5..4c1ff58b 100644 --- a/component_catalog/templates/component_catalog/includes/scan_summary_to_package_modal.html +++ b/component_catalog/templates/component_catalog/includes/scan_summary_to_package_modal.html @@ -20,13 +20,13 @@
Set values from Scan Summary to Package
\ No newline at end of file diff --git a/dje/views.py b/dje/views.py index 1e066aab..70cd0061 100644 --- a/dje/views.py +++ b/dje/views.py @@ -914,10 +914,12 @@ def tab_license(self): tab_fields = [ TabField("license_expression", source=licence_expression_source), - TabField("declared_license_expression"), - TabField("other_license_expression"), ] + for license_field in ["declared_license_expression", "other_license_expression"]: + if hasattr(obj, license_field): + tab_fields.append(TabField(license_field)) + if getattr(obj, "reference_notes", False): tab_fields.append(TabField("reference_notes")) From 53ee8e8d06b5bd4882c12f1d329337b6a5a625b7 Mon Sep 17 00:00:00 2001 From: tdruez Date: Wed, 19 Jun 2024 17:10:10 +0400 Subject: [PATCH 12/27] Add license expression builder for new field on Component/Package forms #63 Signed-off-by: tdruez --- dje/templates/object_form.html | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/dje/templates/object_form.html b/dje/templates/object_form.html index c870acee..f7493a47 100644 --- a/dje/templates/object_form.html +++ b/dje/templates/object_form.html @@ -53,7 +53,17 @@

{% if form.fields.license_expression and licenses_data_for_builder %} let licenses_data_for_builder = JSON.parse(document.getElementById("licenses_data_for_builder").textContent); - setup_awesomplete_builder($('#id_license_expression').get(0), null, 100,licenses_data_for_builder); + const licenseExpressionInputIds = [ + 'id_license_expression', + 'id_declared_license_expression', + 'id_other_license_expression' + ]; + licenseExpressionInputIds.forEach(function(id) { + const licenseExpressionInput = document.getElementById(id); + if (licenseExpressionInput) { + setup_awesomplete_builder(licenseExpressionInput, null, 100, licenses_data_for_builder); + } + }); {% endif %} {% if form.fields.keywords %} From 4d589768dc8d50bfdaab2f591a19ac04cf06b412 Mon Sep 17 00:00:00 2001 From: tdruez Date: Thu, 27 Jun 2024 08:59:52 +0400 Subject: [PATCH 13/27] Refine the update_from_scan logic and tests #63 Signed-off-by: tdruez --- component_catalog/tests/test_scancodeio.py | 63 +- .../bulma-1.0.1-scancode.io-summary.json | 857 ++++++++++++++++++ dejacode_toolkit/scancodeio.py | 4 +- 3 files changed, 883 insertions(+), 41 deletions(-) create mode 100644 component_catalog/tests/testfiles/summary/bulma-1.0.1-scancode.io-summary.json diff --git a/component_catalog/tests/test_scancodeio.py b/component_catalog/tests/test_scancodeio.py index 7bb9b724..172de7fe 100644 --- a/component_catalog/tests/test_scancodeio.py +++ b/component_catalog/tests/test_scancodeio.py @@ -6,6 +6,8 @@ # See https://aboutcode.org for more information about AboutCode FOSS projects. # +import json +from pathlib import Path from unittest import mock from django.test import TestCase @@ -26,6 +28,8 @@ class ScanCodeIOTestCase(TestCase): + data = Path(__file__).parent / "testfiles" + def setUp(self): self.dataspace = Dataspace.objects.create(name="Dataspace") self.basic_user = create_user("basic_user", self.dataspace) @@ -164,6 +168,8 @@ def test_scancodeio_find_project(self, mock_request_get): @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.get_scan_results") @mock.patch("dejacode_toolkit.scancodeio.ScanCodeIO.fetch_scan_data") def test_scancodeio_update_from_scan(self, mock_fetch_scan_data, mock_get_scan_results): + self.package1.license_expression = "" + self.package1.save() scancodeio = ScanCodeIO(self.basic_user) mock_get_scan_results.return_value = None @@ -180,66 +186,43 @@ def test_scancodeio_update_from_scan(self, mock_fetch_scan_data, mock_get_scan_r updated_fields = scancodeio.update_from_scan(self.package1, self.super_user) self.assertEqual([], updated_fields) - mock_fetch_scan_data.return_value = { - "declared_license_expression": "mit", - "declared_holder": "Jeremy Thomas", - "primary_language": "JavaScript", - "key_files": [ - { - "name": "about.NOTICE", - "content": "Notice text", - } - ], - "key_files_packages": [ - { - "purl": "pkg:npm/bulma@0.9.4", - "type": "npm", - "namespace": "", - "name": "bulma", - "version": "0.9.4", - "qualifiers": "", - "subpath": "", - "primary_language": "JavaScript_from_package", - "description": "Modern CSS framework", - "release_date": None, - "homepage_url": "https://bulma.io", - "bug_tracking_url": "https://github.com/jgthms/bulma/issues", - "code_view_url": "", - "vcs_url": "git+https://github.com/jgthms/bulma.git", - "copyright": "", - "license_expression": "mit", - "notice_text": "", - "dependencies": [], - "keywords": ["css", "sass", "flexbox", "responsive", "framework"], - } - ], - } + scan_summary_location = self.data / "summary" / "bulma-1.0.1-scancode.io-summary.json" + with open(scan_summary_location) as f: + scan_summary = json.load(f) + + mock_fetch_scan_data.return_value = scan_summary updated_fields = scancodeio.update_from_scan(self.package1, self.super_user) expected = [ + "license_expression", + "declared_license_expression", "holder", "primary_language", + "other_license_expression", "description", "homepage_url", "keywords", "copyright", - "notice_text", ] self.assertEqual(expected, updated_fields) self.package1.refresh_from_db() + self.assertEqual("mit", self.package1.license_expression) + self.assertEqual("mit", self.package1.declared_license_expression) + self.assertEqual("apache-2.0", self.package1.other_license_expression) self.assertEqual("Jeremy Thomas", self.package1.holder) - self.assertEqual("JavaScript_from_package", self.package1.primary_language) - self.assertEqual("Modern CSS framework", self.package1.description) + self.assertEqual("JavaScript", self.package1.primary_language) + self.assertEqual("Modern CSS framework based on Flexbox", self.package1.description) self.assertEqual("https://bulma.io", self.package1.homepage_url) self.assertEqual("Copyright Jeremy Thomas", self.package1.copyright) - expected_keywords = ["css", "sass", "flexbox", "responsive", "framework"] + expected_keywords = ["css", "sass", "scss", "flexbox", "grid", "responsive", "framework"] self.assertEqual(expected_keywords, self.package1.keywords) self.assertEqual(self.super_user, self.package1.last_modified_by) history_entry = History.objects.get_for_object(self.package1).get() expected = ( - "Automatically updated holder, primary_language, description, " - "homepage_url, keywords, copyright, notice_text from scan results" + "Automatically updated license_expression, declared_license_expression, holder, " + "primary_language, other_license_expression, description, homepage_url, " + "keywords, copyright from scan results" ) self.assertEqual(expected, history_entry.change_message) diff --git a/component_catalog/tests/testfiles/summary/bulma-1.0.1-scancode.io-summary.json b/component_catalog/tests/testfiles/summary/bulma-1.0.1-scancode.io-summary.json new file mode 100644 index 00000000..430d77d7 --- /dev/null +++ b/component_catalog/tests/testfiles/summary/bulma-1.0.1-scancode.io-summary.json @@ -0,0 +1,857 @@ +{ + "declared_license_expression": "mit", + "license_clarity_score": { + "score": 100, + "declared_license": true, + "identification_precision": true, + "has_license_text": true, + "declared_copyrights": true, + "conflicting_license_categories": false, + "ambiguous_compound_licensing": false + }, + "declared_holder": "Jeremy Thomas", + "primary_language": "JavaScript", + "other_license_expressions": [ + { + "value": null, + "count": 78 + } + ], + "other_holders": [ + { + "value": null, + "count": 95 + } + ], + "other_languages": [ + { + "value": "SCSS", + "count": 79 + }, + { + "value": "CSS", + "count": 10 + } + ], + "license_matches": { + "mit": { + "bulma/bulma.scss": [ + { + "license_expression": "mit", + "identifier": "mit-9967e727-165e-9bb5-f090-7de5e47a3929", + "matches": [ + { + "license_expression": "mit", + "matched_text": "/*! bulma.io v1.0.1 | MIT License | github.com/jgthms/bulma */" + } + ] + } + ], + "bulma/css/bulma.css": [ + { + "license_expression": "mit", + "identifier": "mit-9967e727-165e-9bb5-f090-7de5e47a3929", + "matches": [ + { + "license_expression": "mit", + "matched_text": "/*! bulma.io v1.0.1 | MIT License | github.com/jgthms/bulma */" + } + ] + }, + { + "license_expression": "mit", + "identifier": "mit-9967e727-165e-9bb5-f090-7de5e47a3929", + "matches": [ + { + "license_expression": "mit", + "matched_text": "/*! minireset.css v0.0.6 | MIT License | github.com/jgthms/minireset.css */" + } + ] + } + ], + "bulma/css/bulma.min.css": [ + { + "license_expression": "mit", + "identifier": "mit-c50f9352-032a-ddb1-9025-a7ee1bf1bc68", + "matches": [ + { + "license_expression": "mit", + "matched_text": "MIT License |" + }, + { + "license_expression": "mit", + "matched_text": "MIT License |" + } + ] + } + ], + "bulma/css/versions/bulma-no-dark-mode.css": [ + { + "license_expression": "mit", + "identifier": "mit-9967e727-165e-9bb5-f090-7de5e47a3929", + "matches": [ + { + "license_expression": "mit", + "matched_text": "MIT License |" + } + ] + }, + { + "license_expression": "mit", + "identifier": "mit-9967e727-165e-9bb5-f090-7de5e47a3929", + "matches": [ + { + "license_expression": "mit", + "matched_text": "MIT License |" + } + ] + } + ], + "bulma/css/versions/bulma-no-dark-mode.min.css": [ + { + "license_expression": "mit", + "identifier": "mit-c50f9352-032a-ddb1-9025-a7ee1bf1bc68", + "matches": [ + { + "license_expression": "mit", + "matched_text": "MIT License |" + }, + { + "license_expression": "mit", + "matched_text": "MIT License |" + } + ] + } + ], + "bulma/css/versions/bulma-no-helpers-prefixed.css": [ + { + "license_expression": "mit", + "identifier": "mit-9967e727-165e-9bb5-f090-7de5e47a3929", + "matches": [ + { + "license_expression": "mit", + "matched_text": "MIT License |" + } + ] + }, + { + "license_expression": "mit", + "identifier": "mit-9967e727-165e-9bb5-f090-7de5e47a3929", + "matches": [ + { + "license_expression": "mit", + "matched_text": "MIT License |" + } + ] + } + ], + "bulma/css/versions/bulma-no-helpers-prefixed.min.css": [ + { + "license_expression": "mit", + "identifier": "mit-c50f9352-032a-ddb1-9025-a7ee1bf1bc68", + "matches": [ + { + "license_expression": "mit", + "matched_text": "MIT License |" + }, + { + "license_expression": "mit", + "matched_text": "MIT License |" + } + ] + } + ], + "bulma/css/versions/bulma-no-helpers.css": [ + { + "license_expression": "mit", + "identifier": "mit-9967e727-165e-9bb5-f090-7de5e47a3929", + "matches": [ + { + "license_expression": "mit", + "matched_text": "MIT License |" + } + ] + }, + { + "license_expression": "mit", + "identifier": "mit-9967e727-165e-9bb5-f090-7de5e47a3929", + "matches": [ + { + "license_expression": "mit", + "matched_text": "MIT License |" + } + ] + } + ], + "bulma/css/versions/bulma-no-helpers.min.css": [ + { + "license_expression": "mit", + "identifier": "mit-c50f9352-032a-ddb1-9025-a7ee1bf1bc68", + "matches": [ + { + "license_expression": "mit", + "matched_text": "MIT License |" + }, + { + "license_expression": "mit", + "matched_text": "MIT License |" + } + ] + } + ], + "bulma/css/versions/bulma-prefixed.min.css": [ + { + "license_expression": "mit", + "identifier": "mit-9967e727-165e-9bb5-f090-7de5e47a3929", + "matches": [ + { + "license_expression": "mit", + "matched_text": "/*! bulma.io v1.0.1 | MIT License | github.com/jgthms/bulma */" + } + ] + }, + { + "license_expression": "mit", + "identifier": "mit-9967e727-165e-9bb5-f090-7de5e47a3929", + "matches": [ + { + "license_expression": "mit", + "matched_text": "/*! minireset.css v0.0.6 | MIT License | github.com/jgthms/minireset.css */" + } + ] + } + ], + "bulma/css/versions/bulma-prefixed.min.min.css": [ + { + "license_expression": "mit", + "identifier": "mit-c50f9352-032a-ddb1-9025-a7ee1bf1bc68", + "matches": [ + { + "license_expression": "mit", + "matched_text": "MIT License |" + }, + { + "license_expression": "mit", + "matched_text": "MIT License |" + } + ] + } + ], + "bulma/LICENSE": [ + { + "license_expression": "mit", + "identifier": "mit-86fcf017-3572-9813-b7e8-0a10ec4a120f", + "matches": [ + { + "license_expression": "mit", + "matched_text": "The MIT License (MIT)" + }, + { + "license_expression": "mit", + "matched_text": "Permission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in\nall copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE." + } + ] + } + ], + "bulma/package.json": [ + { + "license_expression": "mit", + "identifier": "mit-3fce6ea2-8abd-6c6b-3ede-a37af7c6efee", + "matches": [ + { + "license_expression": "mit", + "matched_text": " \"license\": \"MIT\"," + } + ] + } + ], + "bulma/README.md": [ + { + "license_expression": "mit", + "identifier": "mit-41aa7391-2954-eeff-a675-96bbbbe95fac", + "matches": [ + { + "license_expression": "mit", + "matched_text": "Code copyright 2023 Jeremy Thomas. Code released under [the MIT license](https://github.com/jgthms/bulma/blob/master/LICENSE)." + } + ] + } + ], + "bulma/sass/base/minireset.scss": [ + { + "license_expression": "mit", + "identifier": "mit-9967e727-165e-9bb5-f090-7de5e47a3929", + "matches": [ + { + "license_expression": "mit", + "matched_text": "/*! minireset.css v0.0.6 | MIT License | github.com/jgthms/minireset.css */" + } + ] + } + ], + "bulma/versions/bulma-no-dark-mode.scss": [ + { + "license_expression": "mit", + "identifier": "mit-9967e727-165e-9bb5-f090-7de5e47a3929", + "matches": [ + { + "license_expression": "mit", + "matched_text": "/*! bulma.io v1.0.1 | MIT License | github.com/jgthms/bulma */" + } + ] + } + ], + "bulma/versions/bulma-no-helpers-prefixed.scss": [ + { + "license_expression": "mit", + "identifier": "mit-9967e727-165e-9bb5-f090-7de5e47a3929", + "matches": [ + { + "license_expression": "mit", + "matched_text": "/*! bulma.io v1.0.1 | MIT License | github.com/jgthms/bulma */" + } + ] + } + ], + "bulma/versions/bulma-no-helpers.scss": [ + { + "license_expression": "mit", + "identifier": "mit-9967e727-165e-9bb5-f090-7de5e47a3929", + "matches": [ + { + "license_expression": "mit", + "matched_text": "/*! bulma.io v1.0.1 | MIT License | github.com/jgthms/bulma */" + } + ] + } + ], + "bulma/versions/bulma-prefixed.scss": [ + { + "license_expression": "mit", + "identifier": "mit-9967e727-165e-9bb5-f090-7de5e47a3929", + "matches": [ + { + "license_expression": "mit", + "matched_text": "/*! bulma.io v1.0.1 | MIT License | github.com/jgthms/bulma */" + } + ] + } + ] + } + }, + "key_files": [ + { + "path": "bulma/LICENSE", + "type": "file", + "name": "LICENSE", + "status": "application-package", + "tag": "", + "extension": "", + "size": 1080, + "md5": "16e628e0d477a68ff9f64b315ca323fe", + "sha1": "d1a6c419a8de920621dd00a1494dadec7c6d295d", + "sha256": "fbe7d8ec18b72eded15d79ca73da5eef9e8e258f5f1aff83e4e993e69752279c", + "sha512": "", + "mime_type": "text/plain", + "file_type": "ASCII text", + "programming_language": "", + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_legal": true, + "is_manifest": false, + "is_readme": false, + "is_top_level": true, + "is_key_file": true, + "detected_license_expression": "mit", + "detected_license_expression_spdx": "MIT", + "license_detections": [ + { + "matches": [ + { + "score": 100.0, + "matcher": "2-aho", + "end_line": 1, + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/mit_26.RULE", + "from_file": "codebase/bulma/LICENSE", + "start_line": 1, + "matched_text": "The MIT License (MIT)", + "match_coverage": 100.0, + "matched_length": 4, + "rule_relevance": 100, + "rule_identifier": "mit_26.RULE", + "license_expression": "mit", + "spdx_license_expression": "MIT" + }, + { + "score": 100.0, + "matcher": "2-aho", + "end_line": 21, + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/mit.LICENSE", + "from_file": "codebase/bulma/LICENSE", + "start_line": 5, + "matched_text": "Permission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in\nall copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE.", + "match_coverage": 100.0, + "matched_length": 161, + "rule_relevance": 100, + "rule_identifier": "mit.LICENSE", + "license_expression": "mit", + "spdx_license_expression": "MIT" + } + ], + "identifier": "mit-86fcf017-3572-9813-b7e8-0a10ec4a120f", + "license_expression": "mit", + "license_expression_spdx": "MIT" + } + ], + "license_clues": [], + "percentage_of_license_text": 97.06, + "compliance_alert": "ok", + "copyrights": [ + { + "end_line": 3, + "copyright": "Copyright (c) 2023 Jeremy Thomas", + "start_line": 3 + } + ], + "holders": [ + { + "holder": "Jeremy Thomas", + "end_line": 3, + "start_line": 3 + } + ], + "authors": [], + "package_data": [], + "for_packages": [ + "pkg:npm/bulma@1.0.1?uuid=1de014bd-4e4a-4f1a-a9d2-f1a4399597cd" + ], + "emails": [], + "urls": [], + "extra_data": {}, + "content": "The MIT License (MIT)\n\nCopyright (c) 2023 Jeremy Thomas\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in\nall copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN\nTHE SOFTWARE.\n" + }, + { + "path": "bulma/README.md", + "type": "file", + "name": "README.md", + "status": "application-package", + "tag": "", + "extension": ".md", + "size": 14237, + "md5": "315dabfc696c5a7002af4448a78f7361", + "sha1": "c10a9c074f22a03e405df4fe50720507d81b0a5e", + "sha256": "673e5347d450e9084674f9a9ee0df67656e548e736be31756fa2ec13c503d26d", + "sha512": "", + "mime_type": "text/html", + "file_type": "HTML document, Unicode text, UTF-8 text", + "programming_language": "", + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_legal": false, + "is_manifest": false, + "is_readme": true, + "is_top_level": true, + "is_key_file": true, + "detected_license_expression": "mit", + "detected_license_expression_spdx": "MIT", + "license_detections": [ + { + "matches": [ + { + "score": 100.0, + "matcher": "2-aho", + "end_line": 139, + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/mit_145.RULE", + "from_file": "codebase/bulma/README.md", + "start_line": 139, + "matched_text": "Code copyright 2023 Jeremy Thomas. Code released under [the MIT license](https://github.com/jgthms/bulma/blob/master/LICENSE).", + "match_coverage": 100.0, + "matched_length": 6, + "rule_relevance": 100, + "rule_identifier": "mit_145.RULE", + "license_expression": "mit", + "spdx_license_expression": "MIT" + } + ], + "identifier": "mit-41aa7391-2954-eeff-a675-96bbbbe95fac", + "license_expression": "mit", + "license_expression_spdx": "MIT" + } + ], + "license_clues": [], + "percentage_of_license_text": 0.46, + "compliance_alert": "ok", + "copyrights": [ + { + "end_line": 139, + "copyright": "copyright 2023 Jeremy Thomas", + "start_line": 139 + } + ], + "holders": [ + { + "holder": "Jeremy Thomas", + "end_line": 139, + "start_line": 139 + } + ], + "authors": [], + "package_data": [], + "for_packages": [ + "pkg:npm/bulma@1.0.1?uuid=1de014bd-4e4a-4f1a-a9d2-f1a4399597cd" + ], + "emails": [], + "urls": [ + { + "url": "https://bulma.io/", + "end_line": 1, + "start_line": 1 + }, + { + "url": "https://developer.mozilla.org/en-US/docs/Web/CSS/CSS_Flexible_Box_Layout/Using_CSS_flexible_boxes", + "end_line": 3, + "start_line": 3 + }, + { + "url": "https://img.shields.io/github/v/release/jgthms/bulma?logo=Bulma", + "end_line": 5, + "start_line": 5 + }, + { + "url": "https://img.shields.io/npm/v/bulma.svg", + "end_line": 6, + "start_line": 6 + }, + { + "url": "https://img.shields.io/npm/dm/bulma.svg", + "end_line": 7, + "start_line": 7 + }, + { + "url": "https://data.jsdelivr.com/v1/package/npm/bulma/badge", + "end_line": 8, + "start_line": 8 + }, + { + "url": "https://www.jsdelivr.com/package/npm/bulma", + "end_line": 8, + "start_line": 8 + }, + { + "url": "https://gitter.im/jgthms/bulma", + "end_line": 10, + "start_line": 10 + }, + { + "url": "https://badges.gitter.im/jgthms/bulma.svg", + "end_line": 10, + "start_line": 10 + }, + { + "url": "https://travis-ci.org/jgthms/bulma.svg?branch=master", + "end_line": 11, + "start_line": 11 + }, + { + "url": "https://travis-ci.org/jgthms/bulma", + "end_line": 11, + "start_line": 11 + }, + { + "url": "https://github.com/jgthms/bulma/blob/master/css/bulma.css", + "end_line": 55, + "start_line": 55 + }, + { + "url": "https://bulma.io/documentation/overview/variables", + "end_line": 57, + "start_line": 57 + }, + { + "url": "https://github.com/postcss/autoprefixer", + "end_line": 63, + "start_line": 63 + }, + { + "url": "https://caniuse.com/#feat=flexbox", + "end_line": 63, + "start_line": 63 + }, + { + "url": "https://jekyllrb.com/", + "end_line": 75, + "start_line": 75 + }, + { + "url": "https://bulma.io/documentation/start/overview", + "end_line": 77, + "start_line": 77 + }, + { + "url": "https://github.com/j5bot/bulma-attribute-selectors", + "end_line": 83, + "start_line": 83 + }, + { + "url": "https://github.com/joshuajansen/bulma-rails", + "end_line": 84, + "start_line": 84 + }, + { + "url": "https://github.com/loogn/bulmarazor", + "end_line": 85, + "start_line": 85 + }, + { + "url": "https://github.com/vue-bulma/vue-admin", + "end_line": 86, + "start_line": 86 + }, + { + "url": "https://github.com/jenil/bulmaswatch", + "end_line": 87, + "start_line": 87 + }, + { + "url": "https://github.com/Caiyeon/goldfish", + "end_line": 88, + "start_line": 88 + }, + { + "url": "https://github.com/open-tux/ember-bulma", + "end_line": 89, + "start_line": 89 + }, + { + "url": "https://bloomer.js.org/", + "end_line": 90, + "start_line": 90 + }, + { + "url": "https://github.com/kulakowka/react-bulma", + "end_line": 91, + "start_line": 91 + }, + { + "url": "https://buefy.org/", + "end_line": 92, + "start_line": 92 + }, + { + "url": "https://github.com/vouill/vue-bulma-components", + "end_line": 93, + "start_line": 93 + }, + { + "url": "https://github.com/VizuaaLOG/BulmaJS", + "end_line": 94, + "start_line": 94 + }, + { + "url": "https://github.com/postare/bulma-modal-fx", + "end_line": 95, + "start_line": 95 + }, + { + "url": "https://github.com/groenroos/bulma-stylus", + "end_line": 96, + "start_line": 96 + }, + { + "url": "https://github.com/log1x/bulma.styl", + "end_line": 97, + "start_line": 97 + }, + { + "url": "https://github.com/surprisetalk/elm-bulma", + "end_line": 98, + "start_line": 98 + }, + { + "url": "https://github.com/ahstro/elm-bulma-classes", + "end_line": 99, + "start_line": 99 + }, + { + "url": "https://bulma-customizer.bstash.io/", + "end_line": 100, + "start_line": 100 + }, + { + "url": "https://fulma.github.io/Fulma", + "end_line": 101, + "start_line": 101 + }, + { + "url": "https://github.com/fable-compiler/fable-react", + "end_line": 101, + "start_line": 101 + }, + { + "url": "https://github.com/laravel-enso/enso", + "end_line": 102, + "start_line": 102 + }, + { + "url": "https://github.com/timonweb/django-bulma", + "end_line": 103, + "start_line": 103 + }, + { + "url": "https://github.com/dansup/bulma-templates", + "end_line": 104, + "start_line": 104 + }, + { + "url": "https://github.com/couds/react-bulma-components", + "end_line": 105, + "start_line": 105 + }, + { + "url": "https://github.com/sectore/purescript-bulma", + "end_line": 106, + "start_line": 106 + }, + { + "url": "https://github.com/laravel-enso/vuedatatable", + "end_line": 107, + "start_line": 107 + }, + { + "url": "https://mubaidr.github.io/bulma-fluent", + "end_line": 108, + "start_line": 108 + }, + { + "url": "https://github.com/4d11/csskrt-csskrt", + "end_line": 109, + "start_line": 109 + }, + { + "url": "https://github.com/hipstersmoothie/bulma-pagination-react", + "end_line": 110, + "start_line": 110 + }, + { + "url": "https://github.com/jmaczan/bulma-helpers", + "end_line": 111, + "start_line": 111 + }, + { + "url": "https://github.com/hipstersmoothie/bulma-swatch-hook", + "end_line": 112, + "start_line": 112 + }, + { + "url": "https://github.com/tomhrtly/BulmaWP", + "end_line": 113, + "start_line": 113 + }, + { + "url": "https://github.com/aldi/ralma", + "end_line": 114, + "start_line": 114 + } + ], + "extra_data": {}, + "content": "# [Bulma](https://bulma.io)\n\nBulma is a **modern CSS framework** based on [Flexbox](https://developer.mozilla.org/en-US/docs/Web/CSS/CSS_Flexible_Box_Layout/Using_CSS_flexible_boxes).\n\n![Github](https://img.shields.io/github/v/release/jgthms/bulma?logo=Bulma)\n[![npm](https://img.shields.io/npm/v/bulma.svg)][npm-link]\n[![npm](https://img.shields.io/npm/dm/bulma.svg)][npm-link]\n[![](https://data.jsdelivr.com/v1/package/npm/bulma/badge)](https://www.jsdelivr.com/package/npm/bulma)\n[![Awesome][awesome-badge]][awesome-link]\n[![Join the chat at https://gitter.im/jgthms/bulma](https://badges.gitter.im/jgthms/bulma.svg)](https://gitter.im/jgthms/bulma)\n[![Build Status](https://travis-ci.org/jgthms/bulma.svg?branch=master)](https://travis-ci.org/jgthms/bulma)\n\n\"Bulma:\n\n## Quick install\n\nBulma is constantly in development! Try it out now:\n\n### NPM\n\n```sh\nnpm install bulma\n```\n\n**or**\n\n### Yarn\n\n```sh\nyarn add bulma\n```\n\n### Bower\n\n```sh\nbower install bulma\n```\n\n### Import\n\nAfter installation, you can import the CSS file into your project using this snippet:\n\n```sh\n@import 'bulma/css/bulma.css'\n```\n\n### CDN\n\n[https://www.jsdelivr.com/package/npm/bulma](https://www.jsdelivr.com/package/npm/bulma)\n\nFeel free to raise an issue or submit a pull request.\n\n## CSS only\n\nBulma is a **CSS** framework. As such, the sole output is a single CSS file: [bulma.css](https://github.com/jgthms/bulma/blob/master/css/bulma.css)\n\nYou can either use that file, \"out of the box\", or download the Sass source files to customize the [variables](https://bulma.io/documentation/overview/variables/).\n\nThere is **no** JavaScript included. People generally want to use their own JS implementation (and usually already have one). Bulma can be considered \"environment agnostic\": it's just the style layer on top of the logic.\n\n## Browser Support\n\nBulma uses [autoprefixer](https://github.com/postcss/autoprefixer) to make (most) Flexbox features compatible with earlier browser versions. According to [Can I use](https://caniuse.com/#feat=flexbox), Bulma is compatible with **recent** versions of:\n\n- Chrome\n- Edge\n- Firefox\n- Opera\n- Safari\n\nInternet Explorer (10+) is only partially supported.\n\n## Documentation\n\nThe documentation resides in the [docs](docs) directory, and is built with the Ruby-based [Jekyll](https://jekyllrb.com/) tool.\n\nBrowse the [online documentation here.](https://bulma.io/documentation/start/overview/)\n\n## Related projects\n\n| Project | Description |\n| ------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------- |\n| [Bulma with Attribute Modules](https://github.com/j5bot/bulma-attribute-selectors) | Adds support for attribute-based selectors |\n| [Bulma with Rails](https://github.com/joshuajansen/bulma-rails) | Integrates Bulma with the rails asset pipeline |\n| [BulmaRazor](https://github.com/loogn/bulmarazor) | A lightweight component library based on Bulma and Blazor. |\n| [Vue Admin (dead)](https://github.com/vue-bulma/vue-admin) | Vue Admin framework powered by Bulma |\n| [Bulmaswatch](https://github.com/jenil/bulmaswatch) | Free themes for Bulma |\n| [Goldfish (read-only)](https://github.com/Caiyeon/goldfish) | Vault UI with Bulma, Golang, and Vue Admin |\n| [ember-bulma](https://github.com/open-tux/ember-bulma) | Ember addon providing a collection of UI components for Bulma |\n| [Bloomer](https://bloomer.js.org) | A set of React components for Bulma |\n| [React-bulma](https://github.com/kulakowka/react-bulma) | React.js components for Bulma |\n| [Buefy](https://buefy.org/) | Lightweight UI components for Vue.js based on Bulma |\n| [vue-bulma-components](https://github.com/vouill/vue-bulma-components) | Bulma components for Vue.js with straightforward syntax |\n| [BulmaJS](https://github.com/VizuaaLOG/BulmaJS) | Javascript integration for Bulma. Written in ES6 with a data-\\* API |\n| [Bulma-modal-fx](https://github.com/postare/bulma-modal-fx) | A set of modal window effects with CSS transitions and animations for Bulma |\n| [Bulma Stylus](https://github.com/groenroos/bulma-stylus) | Up-to-date 1:1 translation to Stylus |\n| [Bulma.styl (read-only)](https://github.com/log1x/bulma.styl) | 1:1 Stylus translation of Bulma 0.6.11 |\n| [elm-bulma](https://github.com/surprisetalk/elm-bulma) | Bulma + Elm |\n| [elm-bulma-classes](https://github.com/ahstro/elm-bulma-classes) | Bulma classes prepared for usage with Elm |\n| [Bulma Customizer](https://bulma-customizer.bstash.io/) | Bulma Customizer – Create your own **bespoke** Bulma build |\n| [Fulma](https://fulma.github.io/Fulma/) | Wrapper around Bulma for [fable-react](https://github.com/fable-compiler/fable-react) |\n| [Laravel Enso](https://github.com/laravel-enso/enso) | SPA Admin Panel built with Bulma, VueJS and Laravel |\n| [Django Bulma](https://github.com/timonweb/django-bulma) | Integrates Bulma with Django |\n| [Bulma Templates](https://github.com/dansup/bulma-templates) | Free Templates for Bulma |\n| [React Bulma Components](https://github.com/couds/react-bulma-components) | Another React wrap on React for Bulma.io |\n| [purescript-bulma](https://github.com/sectore/purescript-bulma) | PureScript bindings for Bulma |\n| [Vue Datatable](https://github.com/laravel-enso/vuedatatable) | Bulma themed datatable based on Vue, Laravel & JSON templates |\n| [bulma-fluent](https://mubaidr.github.io/bulma-fluent/) | Fluent Design Theme for Bulma inspired by Microsoft’s Fluent Design System |\n| [csskrt-csskrt](https://github.com/4d11/csskrt-csskrt) | Automatically add Bulma classes to HTML files |\n| [bulma-pagination-react](https://github.com/hipstersmoothie/bulma-pagination-react) | Bulma pagination as a react component |\n| [bulma-helpers](https://github.com/jmaczan/bulma-helpers) | Functional / Atomic CSS classes for Bulma |\n| [bulma-swatch-hook](https://github.com/hipstersmoothie/bulma-swatch-hook) | Bulma swatches as a react hook and a component |\n| [BulmaWP (read-only)](https://github.com/tomhrtly/BulmaWP) | Starter WordPress theme for Bulma |\n| [Ralma](https://github.com/aldi/ralma) | Stateless Ractive.js Components for Bulma |\n| [Django Simple Bulma](https://github.com/python-discord/django-simple-bulma) | Lightweight integration of Bulma and Bulma-Extensions for your Django app |\n| [rbx](https://dfee.github.io/rbx) | Comprehensive React UI Framework written in TypeScript |\n| [Awesome Bulma Templates](https://github.com/aldi/awesome-bulma-templates) | Free real-world Templates built with Bulma |\n| [Trunx](https://github.com/fibo/trunx) | Super Saiyan React components, son of awesome Bulma |\n| [@aybolit/bulma](https://github.com/web-padawan/aybolit/tree/master/packages/bulma) | Web Components library inspired by Bulma and Bulma-extensions |\n| [Drulma](https://www.drupal.org/project/drulma) | Drupal theme for Bulma. |\n| [Bulrush](https://github.com/textbook/bulrush) | A Bulma-based Python Pelican blog theme |\n| [Bulma Variable Export](https://github.com/service-paradis/bulma-variables-export) | Access Bulma Variables in Javascript/Typescript in project using Webpack |\n| [Bulmil](https://github.com/gomah/bulmil) | An agnostic UI components library based on Web Components, made with Bulma & Stencil. |\n| [Svelte Bulma Components](https://github.com/elcobvg/svelte-bulma-components) | Library of UI components to be used in [Svelte.js](https://svelte.technology/) or standalone. |\n| [Bulma Nunjucks Starterkit](https://github.com/benninkcorien/nunjucks-starter-kit) | Starterkit for Nunjucks with Bulma. |\n| [Bulma-Social](https://github.com/aldi/bulma-social) | Social Buttons and Colors for Bulma |\n| [Divjoy](https://divjoy.com/?kit=bulma) | React codebase generator with Bulma templates |\n| [Blazorise](https://github.com/Megabit/Blazorise) | Blazor component library with the support for Bulma CSS framework |\n| [Oruga-Bulma](https://github.com/oruga-ui/theme-bulma) | Bulma theme for [Oruga UI](https://oruga.io) |\n| [@bulvar/bulma](https://github.com/daniil4udo/bulvar/tree/master/packages/bulma) | Bulma with [CSS Variables](https://developer.mozilla.org/en-US/docs/Web/CSS/Using_CSS_custom_properties) support |\n| [@angular-bulma](https://quinnjr.github.io/angular-bulma) | [Angular](https://angular.io/) directives and components to use in your Bulma projects |\n| [Bulma CSS Class Completion](https://github.com/eliutdev/bulma-css-class-completion) | CSS class name completion for the HTML class attribute based on Bulma CSS classes. |\n| [Crispy-Bulma](https://github.com/ckrybus/crispy-bulma) | Bulma template pack for django-crispy-forms |\n| [CASE](https://case.app) | CASE is Lightweight Backend-as-a-Service with essential features: DB, Admin panel, API, JS SDK |\n| [Reactive Bulma](https://github.com/NicolasOmar/reactive-bulma) | A component library based on React, Bulma, Typescript and Rollup |\n\n## Copyright and license ![Github](https://img.shields.io/github/license/jgthms/bulma?logo=Github)\n\nCode copyright 2023 Jeremy Thomas. Code released under [the MIT license](https://github.com/jgthms/bulma/blob/master/LICENSE).\n\n[npm-link]: https://www.npmjs.com/package/bulma\n[awesome-link]: https://github.com/awesome-css-group/awesome-css\n[awesome-badge]: https://cdn.rawgit.com/sindresorhus/awesome/d7305f38d29fed78fa85652e3a63e154dd8e8829/media/badge.svg\n" + } + ], + "key_files_packages": [ + { + "purl": "pkg:npm/bulma@1.0.1", + "type": "npm", + "namespace": "", + "name": "bulma", + "version": "1.0.1", + "qualifiers": "", + "subpath": "", + "tag": "", + "primary_language": "JavaScript", + "description": "Modern CSS framework based on Flexbox", + "release_date": null, + "parties": [ + { + "url": "https://jgthms.com", + "name": "Jeremy Thomas", + "role": "author", + "type": "person", + "email": "bbxdesign@gmail.com" + } + ], + "keywords": [ + "css", + "sass", + "scss", + "flexbox", + "grid", + "responsive", + "framework" + ], + "homepage_url": "https://bulma.io", + "download_url": "https://registry.npmjs.org/bulma/-/bulma-1.0.1.tgz", + "bug_tracking_url": "https://github.com/jgthms/bulma/issues", + "code_view_url": "", + "vcs_url": "git+https://github.com/jgthms/bulma.git", + "repository_homepage_url": "https://www.npmjs.com/package/bulma", + "repository_download_url": "https://registry.npmjs.org/bulma/-/bulma-1.0.1.tgz", + "api_data_url": "https://registry.npmjs.org/bulma/1.0.1", + "size": null, + "md5": "", + "sha1": "", + "sha256": "", + "sha512": "", + "copyright": "", + "holder": "", + "declared_license_expression": "mit", + "declared_license_expression_spdx": "MIT", + "license_detections": [ + { + "matches": [ + { + "score": 100.0, + "matcher": "1-spdx-id", + "end_line": 1, + "rule_url": null, + "from_file": "codebase/bulma/package.json", + "start_line": 1, + "matched_text": "MIT", + "match_coverage": 100.0, + "matched_length": 1, + "rule_relevance": 100, + "rule_identifier": "spdx-license-identifier-mit-5da48780aba670b0860c46d899ed42a0f243ff06", + "license_expression": "mit", + "spdx_license_expression": "MIT" + } + ], + "identifier": "mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf", + "license_expression": "mit", + "license_expression_spdx": "MIT" + } + ], + "other_license_expression": "apache-2.0", + "other_license_expression_spdx": "Apache-2.0", + "other_license_detections": [], + "extracted_license_statement": "- MIT\n", + "compliance_alert": "ok", + "notice_text": "", + "source_packages": [], + "extra_data": {}, + "package_uid": "pkg:npm/bulma@1.0.1?uuid=1de014bd-4e4a-4f1a-a9d2-f1a4399597cd", + "datasource_ids": [ + "npm_package_json" + ], + "datafile_paths": [ + "bulma/package.json" + ], + "file_references": [], + "missing_resources": [], + "modified_resources": [], + "affected_by_vulnerabilities": [] + } + ] +} \ No newline at end of file diff --git a/dejacode_toolkit/scancodeio.py b/dejacode_toolkit/scancodeio.py index a2037eeb..77460134 100644 --- a/dejacode_toolkit/scancodeio.py +++ b/dejacode_toolkit/scancodeio.py @@ -191,7 +191,8 @@ def update_from_scan(self, package, user): logger.debug(f'{self.label}: scan summary not available for package="{package}"') return [] - # 1. Summary fields: declared_license_expression, declared_holder, primary_language + # 1. Summary fields: declared_license_expression, license_expression, + # declared_holder, primary_language for summary_field, model_field in self.AUTO_UPDATE_FIELDS: summary_field_value = scan_summary.get(summary_field) if summary_field_value: @@ -360,6 +361,7 @@ def fetch_project_packages(self, project_uuid): # (scan_field, model_field) AUTO_UPDATE_FIELDS = [ + # declared_license_expression goes in both license fields. ("declared_license_expression", "license_expression"), ("declared_license_expression", "declared_license_expression"), ("other_license_expression", "other_license_expression"), From 3ee264d1d620a27d736b11162f527d9ede799c59 Mon Sep 17 00:00:00 2001 From: tdruez Date: Thu, 27 Jun 2024 12:49:57 +0400 Subject: [PATCH 14/27] Fix rendering of license_expression fields in admin #63 Signed-off-by: tdruez --- component_catalog/forms.py | 12 ++++++++++++ component_catalog/widgets.py | 2 +- dejacode/static/css/dejacode_admin.css | 7 ++++--- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/component_catalog/forms.py b/component_catalog/forms.py index fbb30231..c4e0c12f 100644 --- a/component_catalog/forms.py +++ b/component_catalog/forms.py @@ -949,6 +949,12 @@ class ComponentAdminForm( SetKeywordsChoicesFormMixin, DataspacedAdminForm, ): + expression_field_names = [ + "license_expression", + "declared_license_expression", + "other_license_expression", + ] + keywords = JSONListField( required=False, widget=AdminAwesompleteInputWidget(attrs=autocomplete_placeholder), @@ -983,6 +989,12 @@ class PackageAdminForm( SetKeywordsChoicesFormMixin, DataspacedAdminForm, ): + expression_field_names = [ + "license_expression", + "declared_license_expression", + "other_license_expression", + ] + keywords = JSONListField( required=False, widget=AdminAwesompleteInputWidget(attrs=autocomplete_placeholder), diff --git a/component_catalog/widgets.py b/component_catalog/widgets.py index 849fcba6..31345002 100644 --- a/component_catalog/widgets.py +++ b/component_catalog/widgets.py @@ -26,4 +26,4 @@ def render(self, name, value, *args, **kwargs): self.object_instance.policy_from_primary_license, ) - return rendered + value_from_license + return format_html("{}{}", rendered, value_from_license) diff --git a/dejacode/static/css/dejacode_admin.css b/dejacode/static/css/dejacode_admin.css index 42d313fe..f97b9a0a 100644 --- a/dejacode/static/css/dejacode_admin.css +++ b/dejacode/static/css/dejacode_admin.css @@ -201,15 +201,16 @@ p.url .vURLField {margin-top: 5px; width: 710px;} fieldset.grp-module p.selector-filter label {display: none;} /* Fix for the visibility of the awesomplete license expression builder choices */ -fieldset.grp-module div.license_expression, -fieldset.grp-module div.from_expression, -fieldset.grp-module div.to_expression, +fieldset.grp-module div[class*="_expression"], fieldset.grp-module div.primary_language, fieldset.grp-module div.keywords, fieldset.grp-module div.feature {overflow: visible;} body.mass_update div.awesomplete {display: block;} div.awesomplete > ul {z-index: 1000 !important;} +/* License expression fields */ +textarea#id_declared_license_expression, textarea#id_other_license_expression {height: 60px;} + /* License choices */ code.license_expression {border: 1px solid #ccc; background-color: #EEE; color: black; padding: 2px 4px; border-radius: 3px;} button.submit-inline-button {font-weight: bold; padding: 0 5px; width: auto; background-color: #40A7C9; border: 1px solid #2b8aab;} From d6d4fdc4243197b12e6b677492fd2d18bbbec2cd Mon Sep 17 00:00:00 2001 From: tdruez Date: Thu, 27 Jun 2024 13:14:23 +0400 Subject: [PATCH 15/27] Fix parts of the failing tests #63 Signed-off-by: tdruez --- component_catalog/models.py | 3 +++ component_catalog/tests/test_models.py | 13 ++++++++++--- component_catalog/tests/test_scancodeio.py | 4 +++- component_catalog/tests/test_views.py | 7 ++++--- .../import/component_import_byte_order_mark.csv | 2 +- dejacode_toolkit/scancodeio.py | 2 ++ dje/tests/testfiles/test_dataset_cc_only.json | 7 ++++++- dje/tests/testfiles/test_dataset_pp_only.json | 3 ++- reporting/tests/test_models.py | 10 ++++++++++ 9 files changed, 41 insertions(+), 10 deletions(-) diff --git a/component_catalog/models.py b/component_catalog/models.py index 0d407464..175396c8 100644 --- a/component_catalog/models.py +++ b/component_catalog/models.py @@ -2362,6 +2362,9 @@ def create_from_url(cls, url, user): f"{url} already exists in your Dataspace as {package_link}" ) + if declared_license_expression := package_data.get("declared_license_expression"): + package_data["license_expression"] = declared_license_expression + if package_url: package_data.update(package_url.to_dict(encode=True, empty="")) diff --git a/component_catalog/tests/test_models.py b/component_catalog/tests/test_models.py index 9897d42b..bd7cc815 100644 --- a/component_catalog/tests/test_models.py +++ b/component_catalog/tests/test_models.py @@ -1222,6 +1222,7 @@ def test_component_catalog_models_get_exclude_candidates_fields(self): "reference_notes", "usage_policy", "version", + "other_license_expression", "owner", "release_date", "description", @@ -1248,6 +1249,7 @@ def test_component_catalog_models_get_exclude_candidates_fields(self): "is_notice_in_codebase", "notice_filename", "notice_url", + "declared_license_expression", "dependencies", "codescan_identifier", "website_terms_of_use", @@ -1319,13 +1321,14 @@ def test_component_catalog_models_get_exclude_candidates_fields(self): "copyright", "notice_text", "author", + "declared_license_expression", "dependencies", "repository_homepage_url", "repository_download_url", "api_data_url", - "declared_license", "datasource_id", "file_references", + "other_license_expression", "parties", ], ), @@ -1692,10 +1695,11 @@ def test_package_model_create_from_url_enable_purldb_access( "description": "Abbot Java GUI Test Library", "keywords": ["keyword1", "keyword2"], "homepage_url": "http://abbot.sf.net/", - "download_url": "http://repo1.maven.org/maven2/abbot/abbot/" "1.4.0/abbot-1.4.0.jar", + "download_url": "http://repo1.maven.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0.jar", "size": 687192, "sha1": "a2363646a9dd05955633b450010b59a21af8a423", - "license_expression": "(bsd-new OR eps-1.0 OR apache-2.0 OR mit) AND unknown", + "declared_license_expression": "bsd-new OR epl-1.0 OR apache-2.0", + "declared_license_expression_spdx": "BSD-3-Clause OR EPL-1.0 OR Apache-2.0", "declared_license": "EPL\nhttps://www.eclipse.org/legal/eps-v10.html", "package_url": "pkg:maven/abbot/abbot@1.4.0", } @@ -1704,7 +1708,10 @@ def test_package_model_create_from_url_enable_purldb_access( purl = "pkg:maven/abbot/abbot@1.4.0" package = Package.create_from_url(url=purl, user=self.user) mock_get_purldb_entries.assert_called_once() + self.assertEqual(self.user, package.created_by) + self.assertEqual(purldb_entry["declared_license_expression"], package.license_expression) + for field_name, value in purldb_entry.items(): self.assertEqual(value, getattr(package, field_name)) diff --git a/component_catalog/tests/test_scancodeio.py b/component_catalog/tests/test_scancodeio.py index 172de7fe..08670637 100644 --- a/component_catalog/tests/test_scancodeio.py +++ b/component_catalog/tests/test_scancodeio.py @@ -249,7 +249,7 @@ def test_scancodeio_map_detected_package_data(self): "purl": "pkg:maven/aopalliance/aopalliance@1.0", "primary_language": "Java", "declared_license_expression": "mit AND mit", - "other_license_expression": "apache-20 AND apache-20", + "other_license_expression": "apache-2.0 AND apache-2.0", "keywords": [ "json", "Development Status :: 5 - Production/Stable", @@ -265,6 +265,8 @@ def test_scancodeio_map_detected_package_data(self): "package_url": "pkg:maven/aopalliance/aopalliance@1.0", "purl": "pkg:maven/aopalliance/aopalliance@1.0", "license_expression": "mit", + "declared_license_expression": "mit", + "other_license_expression": "apache-2.0", "primary_language": "Java", "keywords": [ "json", diff --git a/component_catalog/tests/test_views.py b/component_catalog/tests/test_views.py index c67c07f9..6b129d32 100644 --- a/component_catalog/tests/test_views.py +++ b/component_catalog/tests/test_views.py @@ -2599,8 +2599,8 @@ def test_package_details_view_scan_to_package(self, mock_fetch_scan_info, mock_f history = History.objects.get_for_object(self.package1, action_flag=History.CHANGE).get() expected = ( - "Changed Package URL, License expression, Copyright, Primary language, Description, " - "Homepage URL, Release date and Notice text." + "Changed Package URL, Concluded license expression, Copyright, Primary language, " + "Description, Homepage URL, Release date and Notice text." ) self.assertEqual(expected, history.get_change_message()) @@ -2695,7 +2695,7 @@ def test_package_details_view_scan_summary_to_package( self.assertEqual("The Rust Project Developers", self.package1.holder) history = History.objects.get_for_object(self.package1, action_flag=History.CHANGE).get() - expected = "Changed License expression, Primary language and Holder." + expected = "Concluded license expression, Primary language and Holder." self.assertEqual(expected, history.get_change_message()) response = self.client.post(url, post_data, follow=True) @@ -3471,6 +3471,7 @@ def test_component_catalog_package_add_view_initial_data( "primary_language": "Java", "description": "Abbot Java GUI Test Library", "license_expression": "bsd-new OR eps-1.0 OR apache-2.0 OR mit", + "declared_license_expression": "bsd-new OR eps-1.0 OR apache-2.0 OR mit", } self.assertEqual(expected, response.context["form"].initial) diff --git a/component_catalog/tests/testfiles/import/component_import_byte_order_mark.csv b/component_catalog/tests/testfiles/import/component_import_byte_order_mark.csv index f2bf0359..06fde458 100644 --- a/component_catalog/tests/testfiles/import/component_import_byte_order_mark.csv +++ b/component_catalog/tests/testfiles/import/component_import_byte_order_mark.csv @@ -1,2 +1,2 @@ name,reference_notes,usage_policy,request_count,version,owner,release_date,description,copyright,homepage_url,vcs_url,code_view_url,bug_tracking_url,primary_language,admin_notes,notice_text,license_expression,configuration_status,type,approval_reference,guidance,is_active,curation_level,is_license_notice,is_copyright_notice,is_notice_in_codebase,notice_filename,notice_url,dependencies,project,codescan_identifier,website_terms_of_use,ip_sensitivity_approved,affiliate_obligations,affiliate_obligation_triggers,legal_comments,sublicense_allowed,legal_reviewed,distribution_formats_allowed,acceptable_linkages,export_restrictions,approved_download_location,approved_community_interaction,keyword -Partita,test ref notes,,,5.1.3,Unspecified,4/1/19,test desc,Copyright somebody,https://en.wikipedia.org/wiki/Blaise_Pascal,,,,Pascal,,test notice text,mit AND other-permissive,,,,,Yes,35,Yes,Yes,Yes,test.txt,https://en.wikipedia.org/wiki/Blaise_Pascal,,test,,,,,,,,,,,,,,,Framework \ No newline at end of file +Partita,test ref notes,,,5.1.3,Unspecified,4/1/19,test desc,Copyright somebody,https://en.wikipedia.org/wiki/Blaise_Pascal,,,,Pascal,,test notice text,mit AND other-permissive,,,,,Yes,35,Yes,Yes,Yes,test.txt,https://en.wikipedia.org/wiki/Blaise_Pascal,,test,,,,,,,,,,,,,, \ No newline at end of file diff --git a/dejacode_toolkit/scancodeio.py b/dejacode_toolkit/scancodeio.py index 77460134..95ff690e 100644 --- a/dejacode_toolkit/scancodeio.py +++ b/dejacode_toolkit/scancodeio.py @@ -391,6 +391,8 @@ def map_detected_package_data(cls, detected_package): elif scan_data_field.endswith("license_expression"): value = str(Licensing().dedup(value)) + if scan_data_field == "declared_license_expression": + package_data_for_model["license_expression"] = value package_data_for_model[scan_data_field] = value diff --git a/dje/tests/testfiles/test_dataset_cc_only.json b/dje/tests/testfiles/test_dataset_cc_only.json index c209c233..cb14fe0a 100644 --- a/dje/tests/testfiles/test_dataset_cc_only.json +++ b/dje/tests/testfiles/test_dataset_cc_only.json @@ -44,6 +44,8 @@ "last_modified_date": "2011-08-24T09:20:01Z", "reference_notes": "", "usage_policy": null, + "declared_license_expression": "", + "other_license_expression": "", "holder": "", "keywords": [ "Framework" @@ -112,6 +114,8 @@ "last_modified_date": "2011-08-24T09:20:01Z", "reference_notes": "", "usage_policy": null, + "declared_license_expression": "", + "other_license_expression": "", "holder": "", "keywords": [ "Framework" @@ -276,6 +280,8 @@ "version": "", "qualifiers": "", "subpath": "", + "declared_license_expression": "", + "other_license_expression": "", "holder": "", "keywords": [], "cpe": "", @@ -303,7 +309,6 @@ "repository_homepage_url": "", "repository_download_url": "", "api_data_url": "", - "declared_license": "", "datasource_id": "", "file_references": [], "parties": [] diff --git a/dje/tests/testfiles/test_dataset_pp_only.json b/dje/tests/testfiles/test_dataset_pp_only.json index 1b9f656e..fab48776 100644 --- a/dje/tests/testfiles/test_dataset_pp_only.json +++ b/dje/tests/testfiles/test_dataset_pp_only.json @@ -16,6 +16,8 @@ "version": "", "qualifiers": "", "subpath": "", + "declared_license_expression": "", + "other_license_expression": "", "holder": "", "keywords": [], "cpe": "", @@ -43,7 +45,6 @@ "repository_homepage_url": "", "repository_download_url": "", "api_data_url": "", - "declared_license": "", "datasource_id": "", "file_references": [], "parties": [] diff --git a/reporting/tests/test_models.py b/reporting/tests/test_models.py index a1dac023..e1b7455a 100644 --- a/reporting/tests/test_models.py +++ b/reporting/tests/test_models.py @@ -1943,6 +1943,11 @@ def test_get_model_data_for_component_column_template(self): {"group": "Direct Fields", "value": "created_by", "label": "created_by >>"}, {"group": "Direct Fields", "label": "created_date", "value": "created_date"}, {"group": "Direct Fields", "label": "curation_level", "value": "curation_level"}, + { + "group": "Direct Fields", + "label": "declared_license_expression", + "value": "declared_license_expression", + }, {"group": "Direct Fields", "label": "dependencies", "value": "dependencies"}, {"group": "Direct Fields", "label": "description", "value": "description"}, { @@ -2000,6 +2005,11 @@ def test_get_model_data_for_component_column_template(self): {"group": "Direct Fields", "label": "notice_filename", "value": "notice_filename"}, {"group": "Direct Fields", "label": "notice_text", "value": "notice_text"}, {"group": "Direct Fields", "label": "notice_url", "value": "notice_url"}, + { + "group": "Direct Fields", + "label": "other_license_expression", + "value": "other_license_expression", + }, {"group": "Direct Fields", "label": "owner >>", "value": "owner"}, {"group": "Direct Fields", "label": "primary_language", "value": "primary_language"}, {"group": "Direct Fields", "label": "project", "value": "project"}, From 178a12b5435387dd9231012eb6ac8f21f61f300c Mon Sep 17 00:00:00 2001 From: tdruez Date: Thu, 27 Jun 2024 14:07:15 +0400 Subject: [PATCH 16/27] Fix more of the failing tests #63 Signed-off-by: tdruez --- component_catalog/models.py | 1 + component_catalog/tests/test_views.py | 20 +++++++++++--------- dje/tests/test_views.py | 2 +- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/component_catalog/models.py b/component_catalog/models.py index 175396c8..1d6eb2e3 100644 --- a/component_catalog/models.py +++ b/component_catalog/models.py @@ -2362,6 +2362,7 @@ def create_from_url(cls, url, user): f"{url} already exists in your Dataspace as {package_link}" ) + # Duplicate the declared_license_expression into the license_expression field. if declared_license_expression := package_data.get("declared_license_expression"): package_data["license_expression"] = declared_license_expression diff --git a/component_catalog/tests/test_views.py b/component_catalog/tests/test_views.py index 6b129d32..c930cf0b 100644 --- a/component_catalog/tests/test_views.py +++ b/component_catalog/tests/test_views.py @@ -189,7 +189,6 @@ def test_component_catalog_detail_view_content(self): self.assertContains(response, 'id="tab_hierarchy"') # Legal tab is only displayed if one a the legal field is set - self.assertNotContains(response, 'id="tab_legal"') self.component1.legal_comments = "Comments" self.component1.save() response = self.client.get(url) @@ -2136,8 +2135,8 @@ def test_package_details_view_scan_tab_scan_success( expected_declared_license = """ - - l1 + l1 AND (l2 WITH e) @@ -2149,7 +2148,7 @@ def test_package_details_view_scan_tab_scan_success( ' C++' ) expected_other_licenses = """ - mit 3 """ expected_other_holders = ( @@ -2450,12 +2449,14 @@ def test_package_details_view_scan_tab_license_matches( } expected1 = ( - ' apache-2.0 3' "" ) expected2 = ( - ' mit 2' ) response = self.client.get(self.package1_tab_scan_url) @@ -2695,7 +2696,7 @@ def test_package_details_view_scan_summary_to_package( self.assertEqual("The Rust Project Developers", self.package1.holder) history = History.objects.get_for_object(self.package1, action_flag=History.CHANGE).get() - expected = "Concluded license expression, Primary language and Holder." + expected = "Changed Concluded license expression, Primary language and Holder." self.assertEqual(expected, history.get_change_message()) response = self.client.post(url, post_data, follow=True) @@ -2738,7 +2739,8 @@ def test_package_details_view_scan_summary_to_package_libc( response = self.client.get(self.package1_tab_scan_url) expected_license_expression_html = ( '' - ' ' + ' ' "apache-2.0 OR mit" ) expected_holder_html = ( @@ -2790,7 +2792,7 @@ def test_package_details_view_scan_summary_to_package_libc( self.assertEqual(expected_holder, self.package1.holder) history = History.objects.get_for_object(self.package1, action_flag=History.CHANGE).get() - expected = "Changed License expression, Primary language and Holder." + expected = "Changed Concluded license expression, Primary language and Holder." self.assertEqual(expected, history.get_change_message()) response = self.client.post(url, post_data, follow=True) diff --git a/dje/tests/test_views.py b/dje/tests/test_views.py index 040d63f7..0eb94281 100644 --- a/dje/tests/test_views.py +++ b/dje/tests/test_views.py @@ -534,7 +534,7 @@ def test_manage_copy_defaults_view(self): '', '', + ' id="id_form-3-component_46" checked />', '', ' Date: Thu, 27 Jun 2024 14:22:50 +0400 Subject: [PATCH 17/27] Add support for new fields in "add component from package" #63 Signed-off-by: tdruez --- component_catalog/tests/test_views.py | 4 ++++ component_catalog/views.py | 2 ++ 2 files changed, 6 insertions(+) diff --git a/component_catalog/tests/test_views.py b/component_catalog/tests/test_views.py index c930cf0b..ae142d3a 100644 --- a/component_catalog/tests/test_views.py +++ b/component_catalog/tests/test_views.py @@ -1504,6 +1504,8 @@ def test_package_list_view_add_to_component_from_package_data(self): name="common_name", version="", homepage_url="https://p2.com", + declared_license_expression="mit", + other_license_expression="mpl", dataspace=self.dataspace, ) component_add_url = reverse("component_catalog:component_add") @@ -1517,6 +1519,8 @@ def test_package_list_view_add_to_component_from_package_data(self): 'class="urlinput form-control" aria-describedby="id_homepage_url_helptext" ' 'id="id_homepage_url">', ) + self.assertContains(response, "mit") + self.assertContains(response, "mpl") def test_package_list_view_usage_policy_availability(self): self.client.login(username=self.super_user.username, password="secret") diff --git a/component_catalog/views.py b/component_catalog/views.py index 4c0b6fe3..882b1288 100644 --- a/component_catalog/views.py +++ b/component_catalog/views.py @@ -1857,6 +1857,8 @@ def get_form_kwargs(self): "primary_language", "cpe", "license_expression", + "declared_license_expression", + "other_license_expression", "keywords", "usage_policy", "copyright", From e2627a2f0e1758e7af803169061b9688361e3087 Mon Sep 17 00:00:00 2001 From: tdruez Date: Thu, 27 Jun 2024 14:34:46 +0400 Subject: [PATCH 18/27] Add license_expression validation for new fields #63 Signed-off-by: tdruez --- component_catalog/forms.py | 10 ++++++++++ component_catalog/license_expression_dje.py | 8 ++++++++ 2 files changed, 18 insertions(+) diff --git a/component_catalog/forms.py b/component_catalog/forms.py index c4e0c12f..09d78e6e 100644 --- a/component_catalog/forms.py +++ b/component_catalog/forms.py @@ -83,6 +83,11 @@ class ComponentForm( DataspacedModelForm, ): default_on_addition_fields = ["configuration_status"] + expression_field_names = [ + "license_expression", + "declared_license_expression", + "other_license_expression", + ] save_as = True clone_m2m_classes = [ ComponentAssignedPackage, @@ -273,6 +278,11 @@ class PackageForm( PackageFieldsValidationMixin, DataspacedModelForm, ): + expression_field_names = [ + "license_expression", + "declared_license_expression", + "other_license_expression", + ] save_as = True color_initial = True diff --git a/component_catalog/license_expression_dje.py b/component_catalog/license_expression_dje.py index 386e0e92..e79dcfa2 100644 --- a/component_catalog/license_expression_dje.py +++ b/component_catalog/license_expression_dje.py @@ -276,6 +276,14 @@ def clean_license_expression(self): # or without `license_expression` value. return self.clean_expression_base(expression) + def clean_declared_license_expression(self): + expression = self.cleaned_data.get("declared_license_expression") + return self.clean_expression_base(expression) + + def clean_other_license_expression(self): + expression = self.cleaned_data.get("other_license_expression") + return self.clean_expression_base(expression) + def clean_expression_base(self, expression): """ Return a normalized license expression string validated against From 7cf635f74dc984b2a22ff63b39a00b9a238e20f9 Mon Sep 17 00:00:00 2001 From: tdruez Date: Thu, 27 Jun 2024 18:48:33 +0400 Subject: [PATCH 19/27] Fix failing tests #63 Signed-off-by: tdruez --- component_catalog/models.py | 8 -------- component_catalog/tests/test_models.py | 2 -- 2 files changed, 10 deletions(-) diff --git a/component_catalog/models.py b/component_catalog/models.py index 1d6eb2e3..2e4e78db 100644 --- a/component_catalog/models.py +++ b/component_catalog/models.py @@ -341,14 +341,6 @@ class LicenseFieldsMixin(models.Model): class Meta: abstract = True - @property - def declared_license_expression_spdx(self): - return "TODO" - - @property - def other_license_expression_spdx(self): - return "TODO" - def get_cyclonedx_properties(instance): """ diff --git a/component_catalog/tests/test_models.py b/component_catalog/tests/test_models.py index bd7cc815..88e98e55 100644 --- a/component_catalog/tests/test_models.py +++ b/component_catalog/tests/test_models.py @@ -1699,8 +1699,6 @@ def test_package_model_create_from_url_enable_purldb_access( "size": 687192, "sha1": "a2363646a9dd05955633b450010b59a21af8a423", "declared_license_expression": "bsd-new OR epl-1.0 OR apache-2.0", - "declared_license_expression_spdx": "BSD-3-Clause OR EPL-1.0 OR Apache-2.0", - "declared_license": "EPL\nhttps://www.eclipse.org/legal/eps-v10.html", "package_url": "pkg:maven/abbot/abbot@1.4.0", } mock_get_purldb_entries.return_value = [purldb_entry] From f4248af1ec3010821152a83ac7d04268ac3600a5 Mon Sep 17 00:00:00 2001 From: tdruez Date: Fri, 28 Jun 2024 10:39:06 +0400 Subject: [PATCH 20/27] Re-enabled keywords in PurlDB to Package #63 Signed-off-by: tdruez --- component_catalog/tests/test_views.py | 2 ++ component_catalog/views.py | 8 ++------ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/component_catalog/tests/test_views.py b/component_catalog/tests/test_views.py index ae142d3a..887b6742 100644 --- a/component_catalog/tests/test_views.py +++ b/component_catalog/tests/test_views.py @@ -3457,6 +3457,7 @@ def test_component_catalog_package_add_view_initial_data( "primary_language": "Java", "description": "Abbot Java GUI Test Library", "declared_license_expression": "bsd-new OR eps-1.0 OR apache-2.0 OR mit", + "keywords": ["keyword1", "keyword2"], } mock_request_get.return_value = { "count": 1, @@ -3469,6 +3470,7 @@ def test_component_catalog_package_add_view_initial_data( response = self.client.get(add_url + "?package_url=pkg:maven/abbot/abbot@1.4.0") expected = { "filename": "abbot-1.4.0.jar", + "keywords": ["keyword1", "keyword2"], "release_date": "2015-09-22", "type": "maven", "namespace": "abbot", diff --git a/component_catalog/views.py b/component_catalog/views.py index 882b1288..40a9ba63 100644 --- a/component_catalog/views.py +++ b/component_catalog/views.py @@ -1920,13 +1920,9 @@ def get_initial(self): initial = super().get_initial() if purldb_entry := self.get_entry_from_purldb(): + # Duplicate the declared_license_expression as the "concluded" license_expression purldb_entry["license_expression"] = purldb_entry.get("declared_license_expression") - model_fields = [ - field.name - for field in Package._meta.get_fields() - # Generic keywords are not supported because of validation - if field.name != "keywords" - ] + model_fields = [field.name for field in Package._meta.get_fields()] initial_from_purldb_entry = { field_name: value for field_name, value in purldb_entry.items() From 4acc992658dc17ba991c885dd44d1513fd295f89 Mon Sep 17 00:00:00 2001 From: tdruez Date: Fri, 28 Jun 2024 10:39:29 +0400 Subject: [PATCH 21/27] Refine the display of license fields on PurlDB tab #63 Signed-off-by: tdruez --- purldb/tests/test_views.py | 5 ++++- purldb/views.py | 5 +---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/purldb/tests/test_views.py b/purldb/tests/test_views.py index 7a02c166..ae56ef65 100644 --- a/purldb/tests/test_views.py +++ b/purldb/tests/test_views.py @@ -51,7 +51,7 @@ "url": null } ], - "keywords": [], + "keywords": ["keyword1", "keyword2"], "homepage_url": "http://abbot.sf.net/", "download_url": "https://repo1.maven.org/maven2/abbot/abbot/1.4.0/abbot-1.4.0.jar", "bug_tracking_url": null, @@ -270,6 +270,9 @@ def test_purldb_details_view_content(self): expected = 'id="tab_purldb"' self.assertContains(response, expected) + expected = '
keyword1\nkeyword2
' + self.assertContains(response, expected, html=True) + def test_purldb_search_table_view(self): search_table_url = reverse("purldb:purldb_search_table") diff --git a/purldb/views.py b/purldb/views.py index b00ff87c..d31bc3da 100644 --- a/purldb/views.py +++ b/purldb/views.py @@ -82,9 +82,6 @@ def get_purldb_tab_fields(purldb_entry, dataspace): if package_url: tab_fields.append(("Package URL", package_url, Package.package_url_help())) - # Workaround the `declared_license_expression` field renaming - sorted_data["license_expression"] = sorted_data.get("declared_license_expression") - for field_name, value in sorted_data.items(): if not value or field_name in exclude: continue @@ -94,7 +91,7 @@ def get_purldb_tab_fields(purldb_entry, dataspace): except FieldDoesNotExist: continue - if field_name == "license_expression": + if field_name == "declared_license_expression": show_policy = dataspace.show_usage_policy_in_user_views licensing = get_licensing_for_formatted_render(dataspace, show_policy) value = format_html(get_formatted_expression(licensing, value, show_policy)) From 9080c2c36df27234cc69bdb15456afa83c91af4b Mon Sep 17 00:00:00 2001 From: tdruez Date: Fri, 28 Jun 2024 12:11:11 +0400 Subject: [PATCH 22/27] Refactor the merge logic of Component from Package merge #63 Signed-off-by: tdruez --- component_catalog/tests/test_views.py | 28 +++++++++ component_catalog/views.py | 90 ++++++++++++++++----------- dje/utils.py | 11 ++++ 3 files changed, 93 insertions(+), 36 deletions(-) diff --git a/component_catalog/tests/test_views.py b/component_catalog/tests/test_views.py index 887b6742..b90b12c5 100644 --- a/component_catalog/tests/test_views.py +++ b/component_catalog/tests/test_views.py @@ -39,6 +39,7 @@ from component_catalog.models import ComponentType from component_catalog.models import Package from component_catalog.models import Subcomponent +from component_catalog.views import ComponentAddView from component_catalog.views import ComponentListView from component_catalog.views import PackageDetailsView from component_catalog.views import PackageTabScanView @@ -1497,6 +1498,7 @@ def test_package_list_view_add_to_component_from_package_data(self): name="common_name", version="1.0", homepage_url="https://p1.com", + dependencies=[{"purl": "pkg:maven/org.apache.lucene/lucene"}], dataspace=self.dataspace, ) p2 = Package.objects.create( @@ -1521,6 +1523,32 @@ def test_package_list_view_add_to_component_from_package_data(self): ) self.assertContains(response, "mit") self.assertContains(response, "mpl") + self.assertNotContains(response, "pkg:maven/org.apache.lucene/lucene") + + component_add_url = reverse("component_catalog:component_add") + url = f"{component_add_url}?package_ids={p1.id}" + response = self.client.get(url) + self.assertContains(response, "pkg:maven/org.apache.lucene/lucene") + + def test_add_component_from_package_data_extract_common_values(self): + extract_common_values = ComponentAddView.extract_common_values + + packages = None + self.assertEqual({}, extract_common_values(packages)) + + packages = [] + self.assertEqual({}, extract_common_values(packages)) + + package1 = {"name": "common", "version": "1.0", "empty_value": ""} + package2 = {"name": "common", "version": "2.0", "list_value": [1], "dict_value": {"a": "b"}} + + packages = [package1] + expected = {"name": "common", "version": "1.0"} + self.assertEqual(expected, extract_common_values(packages)) + + packages = [package1, package2] + expected = {"name": "common"} + self.assertEqual(expected, extract_common_values(packages)) def test_package_list_view_usage_policy_availability(self): self.client.login(username=self.super_user.username, password="secret") diff --git a/component_catalog/views.py b/component_catalog/views.py index 40a9ba63..13a5e6a8 100644 --- a/component_catalog/views.py +++ b/component_catalog/views.py @@ -88,6 +88,7 @@ from dje.utils import get_preserved_filters from dje.utils import is_available from dje.utils import is_uuid4 +from dje.utils import remove_empty_values from dje.utils import str_to_id_list from dje.views import AcceptAnonymousMixin from dje.views import APIWrapperListView @@ -868,8 +869,11 @@ def tab_legal(self): ] fields = self.get_tab_fields(tab_fields) - # At least 1 value need to be set for the tab to be available. - if not any([1 for field in fields if field[1] and field[0] != "License expression"]): + # At least 1 value need to be set (excepting the license_expression) + # for the tab to be available. + if not any( + [1 for field in fields if field[1] and field[0] != "Concluded license expression"] + ): return return {"fields": fields} @@ -1839,57 +1843,71 @@ class ComponentAddView( model = Component form_class = ComponentForm permission_required = "component_catalog.add_component" + initial_from_package_fields = [ + "name", + "version", + "description", + "primary_language", + "cpe", + "license_expression", + "declared_license_expression", + "other_license_expression", + "keywords", + "usage_policy", + "copyright", + "holder", + "notice_text", + "dependencies", + "reference_notes", + "release_date", + "homepage_url", + "code_view_url", + "vcs_url", + "bug_tracking_url", + "project", + ] def get_form_kwargs(self): kwargs = super().get_form_kwargs() - package_ids = self.request.GET.get("package_ids") - package_ids = str_to_id_list(package_ids) + package_ids = str_to_id_list(self.request.GET.get("package_ids", "")) packages = ( Package.objects.scope(self.request.user.dataspace) .filter(id__in=package_ids) - .values( - "id", - "name", - "version", - "description", - "primary_language", - "cpe", - "license_expression", - "declared_license_expression", - "other_license_expression", - "keywords", - "usage_policy", - "copyright", - "holder", - "notice_text", - "dependencies", - "reference_notes", - "release_date", - "homepage_url", - "project", - ) + .values("id", *self.initial_from_package_fields) ) initial = {"packages_ids": ",".join([str(entry.pop("id")) for entry in packages])} + if packages: - for key in packages[0].keys(): - unique_values = set() - for entry in packages: - value = entry.get(key) - if not value: - continue - if isinstance(value, list): - value = ", ".join(value) - unique_values.add(value) - if len(unique_values) == 1: - initial[key] = list(unique_values)[0] + initial.update(self.extract_common_values(packages)) kwargs.update({"initial": initial}) return kwargs + @staticmethod + def extract_common_values(packages): + if not (packages): + return {} + + if len(packages) == 1: + return remove_empty_values(packages[0]) + + common_values = {} + for key in packages[0].keys(): + unique_values = set() + for entry in packages: + value = entry.get(key) + if value in EMPTY_VALUES or isinstance(value, (list, dict)): + continue + unique_values.add(value) + if len(unique_values) == 1: + common_values[key] = list(unique_values)[0] + + return common_values + class ComponentUpdateView( LicenseDataForBuilderMixin, diff --git a/dje/utils.py b/dje/utils.py index 08cb637f..cbf738fd 100644 --- a/dje/utils.py +++ b/dje/utils.py @@ -18,6 +18,7 @@ from django.contrib.auth import get_permission_codename from django.contrib.contenttypes.models import ContentType +from django.core.validators import EMPTY_VALUES from django.db import models from django.db.models.options import Options from django.http.request import HttpRequest @@ -631,3 +632,13 @@ def is_purl_str(url, validate=False): except ValueError: return False return True + + +def remove_empty_values(input_dict): + """ + Return a new dict not including empty value entries from `input_dict`. + + None, empty string, empty list, and empty dict/set are cleaned. + `0` and `False` values are kept. + """ + return {key: value for key, value in input_dict.items() if value not in EMPTY_VALUES} From 3f8e268bd114b1d9d64cf34c4487d6f09b02f6b9 Mon Sep 17 00:00:00 2001 From: tdruez Date: Fri, 28 Jun 2024 13:32:15 +0400 Subject: [PATCH 23/27] Add expression rendering for new license fields #63 Signed-off-by: tdruez --- component_catalog/license_expression_dje.py | 50 ++++++++++++++++++--- component_catalog/views.py | 12 +++-- dejacode/settings.py | 6 +++ dje/views.py | 26 +++++------ license_library/models.py | 16 +++---- purldb/views.py | 6 +-- 6 files changed, 77 insertions(+), 39 deletions(-) diff --git a/component_catalog/license_expression_dje.py b/component_catalog/license_expression_dje.py index e79dcfa2..a3b2af8a 100644 --- a/component_catalog/license_expression_dje.py +++ b/component_catalog/license_expression_dje.py @@ -9,6 +9,7 @@ from collections import defaultdict from itertools import chain +from django.core.cache import caches from django.core.exceptions import ValidationError from django.forms import widgets from django.urls import reverse @@ -23,6 +24,8 @@ from dje.widgets import AwesompleteInputWidgetMixin from license_library.models import License +licensing_cache = caches["licensing"] + def build_licensing(licenses=None): """ @@ -34,6 +37,39 @@ def build_licensing(licenses=None): return Licensing(licenses) +def fetch_licensing_for_dataspace(dataspace, license_keys=None): + """ + Return a Licensing object for the provided ``dataspace``. + An optional list of ``license_keys`` can be provided to limit the licenses + included in the Licensing object. + """ + license_qs = License.objects.scope(dataspace).for_expression(license_keys) + licensing = build_licensing(license_qs) + return licensing + + +def get_dataspace_licensing(dataspace, license_keys=None): + """ + Return a Licensing object for the provided ``dataspace``. + The Licensing object is put in the cache for 5 minutes. + Note that the cache is not used when ``license_keys`` are provided. + """ + if license_keys is not None: + # Bypass cache if license_keys is provided + return fetch_licensing_for_dataspace(dataspace, license_keys) + + cache_key = str({dataspace.name}) + # First look in the cache for an existing Licensing for this Dataspace + licensing = licensing_cache.get(cache_key) + + if licensing is None: + # If not cached, compute the value and cache it + licensing = fetch_licensing_for_dataspace(dataspace, license_keys) + licensing_cache.set(cache_key, licensing, timeout=600) # 10 minutes + + return licensing + + def parse_expression( expression, licenses=None, validate_known=True, validate_strict=False, simple=False ): @@ -381,12 +417,6 @@ def get_unique_license_keys(license_expression): return {symbol.key for symbol in symbols} -def get_licensing_for_formatted_render(dataspace, show_policy=False, license_keys=None): - license_qs = License.objects.scope(dataspace).for_expression(show_policy, license_keys) - licensing = build_licensing(license_qs) - return licensing - - def get_formatted_expression(licensing, license_expression, show_policy, show_category=False): normalized = parse_expression( license_expression, licenses=licensing, validate_known=False, validate_strict=False @@ -394,3 +424,11 @@ def get_formatted_expression(licensing, license_expression, show_policy, show_ca return normalized.render_as_readable( as_link=True, show_policy=show_policy, show_category=show_category ) + + +def render_expression_for_ui(expression_str, dataspace): + show_policy = dataspace.show_usage_policy_in_user_views + licensing = get_dataspace_licensing(dataspace) + + formatted_expression = get_formatted_expression(licensing, expression_str, show_policy) + return format_html(formatted_expression) diff --git a/component_catalog/views.py b/component_catalog/views.py index 13a5e6a8..bcd8374b 100644 --- a/component_catalog/views.py +++ b/component_catalog/views.py @@ -64,8 +64,8 @@ from component_catalog.forms import ScanSummaryToPackageForm from component_catalog.forms import ScanToPackageForm from component_catalog.forms import SetPolicyForm +from component_catalog.license_expression_dje import get_dataspace_licensing from component_catalog.license_expression_dje import get_formatted_expression -from component_catalog.license_expression_dje import get_licensing_for_formatted_render from component_catalog.license_expression_dje import get_unique_license_keys from component_catalog.models import Component from component_catalog.models import Package @@ -2082,7 +2082,7 @@ def _generate_license_match_card(cls, path, detection_data): """ @staticmethod - def _get_licensing_for_formatted_render(dataspace, license_expressions): + def _get_dataspace_licensing(dataspace, license_expressions): # Get the set of unique license symbols as an optimization # to filter the License QuerySet to relevant objects. license_keys = set() @@ -2092,7 +2092,7 @@ def _get_licensing_for_formatted_render(dataspace, license_expressions): license_keys.update(get_unique_license_keys(expression)) show_policy = dataspace.show_usage_policy_in_user_views - licensing = get_licensing_for_formatted_render(dataspace, show_policy, license_keys) + licensing = get_dataspace_licensing(dataspace, license_keys) return licensing, show_policy @@ -2106,9 +2106,7 @@ def get_license_expressions_scan_values( license_matches, checked=False, ): - licensing, show_policy = cls._get_licensing_for_formatted_render( - dataspace, license_expressions - ) + licensing, show_policy = cls._get_dataspace_licensing(dataspace, license_expressions) values = [] for entry in license_expressions: license_expression = entry.get("value") @@ -2267,7 +2265,7 @@ def key_files_fields(self, key_files): key_file["matched_texts"] = matched_texts if detected_license_expression := key_file["detected_license_expression"]: - licensing, show_policy = self._get_licensing_for_formatted_render( + licensing, show_policy = self._get_dataspace_licensing( self.object.dataspace, [{"value": detected_license_expression}] ) key_file["formatted_expression"] = get_formatted_expression( diff --git a/dejacode/settings.py b/dejacode/settings.py index 77e5fef9..f14bc897 100644 --- a/dejacode/settings.py +++ b/dejacode/settings.py @@ -418,6 +418,12 @@ def gettext_noop(s): "LOCATION": REDIS_URL, "TIMEOUT": 900, # 15 minutes, in seconds }, + "licensing": { + "BACKEND": CACHE_BACKEND, + "LOCATION": REDIS_URL, + "TIMEOUT": 300, # 10 minutes, in seconds + "KEY_PREFIX": "licensing", + }, "vulnerabilities": { "BACKEND": CACHE_BACKEND, "LOCATION": REDIS_URL, diff --git a/dje/views.py b/dje/views.py index 70cd0061..f28c6f5c 100644 --- a/dje/views.py +++ b/dje/views.py @@ -37,6 +37,7 @@ from django.contrib.staticfiles import finders from django.core.exceptions import ObjectDoesNotExist from django.core.paginator import Paginator +from django.core.validators import EMPTY_VALUES from django.db import models from django.db.models import Value from django.db.models.functions import Concat @@ -73,6 +74,7 @@ from notifications import views as notifications_views from component_catalog.license_expression_dje import get_license_objects +from component_catalog.license_expression_dje import render_expression_for_ui from dejacode_toolkit.purldb import PurlDB from dejacode_toolkit.scancodeio import ScanCodeIO from dejacode_toolkit.vulnerablecode import VulnerableCode @@ -807,9 +809,8 @@ def get_tab_fields(self, tab_fields): Return list of fields suitable for the tab templates display using a list of `TabField` as input. """ - EMPTY_VALUES = ("", None, []) # Used for conditions without absorbing `False` values. - fields = [] + for tab_field in tab_fields: if not isinstance(tab_field, TabField): fields.append(tab_field) @@ -905,20 +906,18 @@ def tab_components(self): def tab_license(self): """Return a mapping of data for use in the license tab display or None.""" obj = self.object - licenses = get_license_objects(obj.license_expression, obj.licensing) - show_usage_policy = self.request.user.dataspace.show_usage_policy_in_user_views - - licence_expression_source = "license_expression_linked" - if show_usage_policy: - licence_expression_source = "get_license_expression_linked_with_policy" + tab_fields = [] - tab_fields = [ - TabField("license_expression", source=licence_expression_source), + license_fields = [ + "license_expression", + "declared_license_expression", + "other_license_expression", ] - for license_field in ["declared_license_expression", "other_license_expression"]: - if hasattr(obj, license_field): - tab_fields.append(TabField(license_field)) + for license_field in license_fields: + if expression := getattr(obj, license_field, None): + rendered_expression = render_expression_for_ui(expression, obj.dataspace) + tab_fields.append(TabField(license_field, value=rendered_expression)) if getattr(obj, "reference_notes", False): tab_fields.append(TabField("reference_notes")) @@ -935,6 +934,7 @@ def tab_license(self): ) licenses_per_table = 4 + licenses = get_license_objects(obj.license_expression, obj.licensing) licenses_tables = chunked(licenses, chunk_size=licenses_per_table) help_texts = { diff --git a/license_library/models.py b/license_library/models.py index 99d0fc04..37f4bee8 100644 --- a/license_library/models.py +++ b/license_library/models.py @@ -464,20 +464,16 @@ def render(self, template, as_link=False, show_policy=False, **kwargs): class LicenseQuerySet(DataspacedQuerySet): - def for_expression(self, show_policy=False, license_keys=None): - select_related = ["dataspace"] - only = [ + def for_expression(self, license_keys=None): + qs = self.only( "key", + "name", "short_name", + "spdx_license_key", "is_exception", + "usage_policy", "dataspace", - ] - - if show_policy: - select_related.append("usage_policy") - only.append("usage_policy") - - qs = self.select_related(*select_related).only(*only) + ).select_related("dataspace", "usage_policy") if license_keys: qs = qs.filter(key__in=license_keys) diff --git a/purldb/views.py b/purldb/views.py index d31bc3da..1c383452 100644 --- a/purldb/views.py +++ b/purldb/views.py @@ -25,8 +25,8 @@ import saneyaml from crispy_forms.helper import FormHelper +from component_catalog.license_expression_dje import get_dataspace_licensing from component_catalog.license_expression_dje import get_formatted_expression -from component_catalog.license_expression_dje import get_licensing_for_formatted_render from component_catalog.license_expression_dje import get_unique_license_keys from dejacode_toolkit.purldb import PurlDB from dje.templatetags.dje_tags import urlize_target_blank @@ -93,7 +93,7 @@ def get_purldb_tab_fields(purldb_entry, dataspace): if field_name == "declared_license_expression": show_policy = dataspace.show_usage_policy_in_user_views - licensing = get_licensing_for_formatted_render(dataspace, show_policy) + licensing = get_dataspace_licensing(dataspace) value = format_html(get_formatted_expression(licensing, value, show_policy)) elif field_name == "dependencies": value = json.dumps(value, indent=2) @@ -132,7 +132,7 @@ def inject_license_expression_formatted(dataspace, object_list): if expression: license_keys.update(get_unique_license_keys(expression)) - licensing = get_licensing_for_formatted_render(dataspace, show_policy, license_keys) + licensing = get_dataspace_licensing(dataspace, license_keys) for obj in object_list: expression = obj.get("declared_license_expression") From 111edd7227fc17434924da16180b1ddee49a4b8f Mon Sep 17 00:00:00 2001 From: tdruez Date: Fri, 28 Jun 2024 14:20:39 +0400 Subject: [PATCH 24/27] Add support for rendering any expression as SPDX #63 Signed-off-by: tdruez --- component_catalog/license_expression_dje.py | 12 ++++++++++-- component_catalog/models.py | 20 ++++++++++++++++++++ dje/views.py | 4 ++-- reporting/forms.py | 6 ++++++ reporting/tests/test_models.py | 15 +++++++++++++++ 5 files changed, 53 insertions(+), 4 deletions(-) diff --git a/component_catalog/license_expression_dje.py b/component_catalog/license_expression_dje.py index a3b2af8a..6e347314 100644 --- a/component_catalog/license_expression_dje.py +++ b/component_catalog/license_expression_dje.py @@ -426,9 +426,17 @@ def get_formatted_expression(licensing, license_expression, show_policy, show_ca ) -def render_expression_for_ui(expression_str, dataspace): +def render_expression_as_html(expression, dataspace): + """Return the ``expression`` as rendered HTML content.""" show_policy = dataspace.show_usage_policy_in_user_views licensing = get_dataspace_licensing(dataspace) - formatted_expression = get_formatted_expression(licensing, expression_str, show_policy) + formatted_expression = get_formatted_expression(licensing, expression, show_policy) return format_html(formatted_expression) + + +def get_expression_as_spdx(expression, dataspace): + """Return an SPDX license expression built from the ``expression``.""" + licensing = get_dataspace_licensing(dataspace) + parsed_expression = parse_expression(expression, licensing) + return parsed_expression.render(template="{symbol.spdx_license_key}") diff --git a/component_catalog/models.py b/component_catalog/models.py index 2e4e78db..c2246ea5 100644 --- a/component_catalog/models.py +++ b/component_catalog/models.py @@ -44,6 +44,7 @@ from packageurl.contrib.django.utils import without_empty_values from component_catalog.license_expression_dje import build_licensing +from component_catalog.license_expression_dje import get_expression_as_spdx from component_catalog.license_expression_dje import get_license_objects from component_catalog.license_expression_dje import parse_expression from dejacode_toolkit import spdx @@ -196,6 +197,9 @@ def get_license_expression_spdx_id(self): "LicenseRef-". See discussion at https://github.com/spdx/tools-java/issues/73 + + Note: A new get_expression_as_spdx function is available and should be used in + place of this one. """ expression = self.get_license_expression("{symbol.spdx_id}") if expression: @@ -215,6 +219,14 @@ def _get_primary_license(self): primary_license = cached_property(_get_primary_license) + def get_expression_as_spdx(self, expression): + if expression: + return get_expression_as_spdx(expression, self.dataspace) + + @property + def concluded_license_expression_spdx(self): + return self.get_expression_as_spdx(self.license_expression) + def save(self, *args, **kwargs): """ Call the handle_assigned_licenses method on save, except during copy. @@ -341,6 +353,14 @@ class LicenseFieldsMixin(models.Model): class Meta: abstract = True + @property + def declared_license_expression_spdx(self): + return self.get_expression_as_spdx(self.declared_license_expression) + + @property + def other_license_expression_spdx(self): + return self.get_expression_as_spdx(self.other_license_expression) + def get_cyclonedx_properties(instance): """ diff --git a/dje/views.py b/dje/views.py index f28c6f5c..db20bb36 100644 --- a/dje/views.py +++ b/dje/views.py @@ -74,7 +74,7 @@ from notifications import views as notifications_views from component_catalog.license_expression_dje import get_license_objects -from component_catalog.license_expression_dje import render_expression_for_ui +from component_catalog.license_expression_dje import render_expression_as_html from dejacode_toolkit.purldb import PurlDB from dejacode_toolkit.scancodeio import ScanCodeIO from dejacode_toolkit.vulnerablecode import VulnerableCode @@ -916,7 +916,7 @@ def tab_license(self): for license_field in license_fields: if expression := getattr(obj, license_field, None): - rendered_expression = render_expression_for_ui(expression, obj.dataspace) + rendered_expression = render_expression_as_html(expression, obj.dataspace) tab_fields.append(TabField(license_field, value=rendered_expression)) if getattr(obj, "reference_notes", False): diff --git a/reporting/forms.py b/reporting/forms.py index 4800274e..243e8a06 100644 --- a/reporting/forms.py +++ b/reporting/forms.py @@ -189,6 +189,9 @@ def get_model_data_for_column_template(dataspace=None): "Properties": [ "urn", "details_url", + "concluded_license_expression_spdx", + "declared_license_expression_spdx", + "other_license_expression_spdx", "primary_license", "attribution_required", "redistribution_required", @@ -201,6 +204,9 @@ def get_model_data_for_column_template(dataspace=None): "identifier", "urn", "details_url", + "concluded_license_expression_spdx", + "declared_license_expression_spdx", + "other_license_expression_spdx", "primary_license", "attribution_required", "redistribution_required", diff --git a/reporting/tests/test_models.py b/reporting/tests/test_models.py index e1b7455a..c1552843 100644 --- a/reporting/tests/test_models.py +++ b/reporting/tests/test_models.py @@ -2059,6 +2059,21 @@ def test_get_model_data_for_component_column_template(self): {"group": "Related Fields", "label": "related_parents", "value": "related_parents"}, {"group": "Properties", "label": "urn", "value": "urn"}, {"group": "Properties", "label": "details_url", "value": "details_url"}, + { + "group": "Properties", + "label": "concluded_license_expression_spdx", + "value": "concluded_license_expression_spdx", + }, + { + "group": "Properties", + "label": "declared_license_expression_spdx", + "value": "declared_license_expression_spdx", + }, + { + "group": "Properties", + "label": "other_license_expression_spdx", + "value": "other_license_expression_spdx", + }, {"group": "Properties", "value": "primary_license", "label": "primary_license"}, { "group": "Properties", From f334131a28a6a76a336844c8c17142dadeb25b30 Mon Sep 17 00:00:00 2001 From: tdruez Date: Fri, 28 Jun 2024 17:29:13 +0400 Subject: [PATCH 25/27] Refine get_expression_as_spdx and add unti test #63 Signed-off-by: tdruez --- component_catalog/models.py | 8 +++++- .../tests/test_license_expression_dje.py | 28 ++++++++++++++++++- component_catalog/tests/test_models.py | 23 +++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/component_catalog/models.py b/component_catalog/models.py index c2246ea5..b03a3c1d 100644 --- a/component_catalog/models.py +++ b/component_catalog/models.py @@ -36,6 +36,7 @@ from cyclonedx.model import component as cyclonedx_component from cyclonedx.model import contact as cyclonedx_contact from cyclonedx.model import license as cyclonedx_license +from license_expression import ExpressionError from packageurl import PackageURL from packageurl.contrib import purl2url from packageurl.contrib import url2purl @@ -220,8 +221,13 @@ def _get_primary_license(self): primary_license = cached_property(_get_primary_license) def get_expression_as_spdx(self, expression): - if expression: + if not expression: + return + + try: return get_expression_as_spdx(expression, self.dataspace) + except ExpressionError as e: + return str(e) @property def concluded_license_expression_spdx(self): diff --git a/component_catalog/tests/test_license_expression_dje.py b/component_catalog/tests/test_license_expression_dje.py index 1add8e18..b44ad47c 100644 --- a/component_catalog/tests/test_license_expression_dje.py +++ b/component_catalog/tests/test_license_expression_dje.py @@ -10,24 +10,42 @@ import os from collections import namedtuple from itertools import zip_longest -from unittest import TestCase from django.core.exceptions import ValidationError +from django.test import TestCase +from license_expression import ExpressionError from license_expression import LicenseSymbolLike from license_expression import ParseError from license_expression import as_symbols from component_catalog.license_expression_dje import build_licensing +from component_catalog.license_expression_dje import get_expression_as_spdx from component_catalog.license_expression_dje import get_license_objects from component_catalog.license_expression_dje import get_unique_license_keys from component_catalog.license_expression_dje import normalize_and_validate_expression from component_catalog.license_expression_dje import parse_expression +from dje.models import Dataspace +from license_library.models import License +from organization.models import Owner MockLicense = namedtuple("MockLicense", "key aliases is_exception") class LicenseExpressionDjeTestCase(TestCase): + def setUp(self): + self.dataspace = Dataspace.objects.create(name="Starship") + self.owner = Owner.objects.create(name="Owner", dataspace=self.dataspace) + + self.license1 = License.objects.create( + key="apache-2.0", + name="Apache License 2.0", + short_name="Apache 2.0", + spdx_license_key="Apache-2.0", + dataspace=self.dataspace, + owner=self.owner, + ) + def test_as_symbols(self): lic1 = MockLicense("x11", ["X11 License"], True) lic2 = MockLicense("x11-xconsortium", ["X11 XConsortium"], False) @@ -263,6 +281,14 @@ def test_get_unique_license_keys(self): expected = {"lgpl", "oracle-bcl", "gps-2.0", "classpath"} self.assertEqual(expected, get_unique_license_keys(expression)) + def test_get_expression_as_spdx(self): + expression = "apache-2.0" + expected = "Apache-2.0" + self.assertEqual(expected, get_expression_as_spdx(expression, self.dataspace)) + + with self.assertRaises(ExpressionError): + get_expression_as_spdx("unknown", self.dataspace) + def _print_sequence_diff(left, right): for lft, rht in zip_longest(left.split(), right.split()): diff --git a/component_catalog/tests/test_models.py b/component_catalog/tests/test_models.py index 88e98e55..e66a14f0 100644 --- a/component_catalog/tests/test_models.py +++ b/component_catalog/tests/test_models.py @@ -443,6 +443,29 @@ def test_component_get_license_expression_spdx_id(self): expected = "SPDX-1 WITH SPDX-2" self.assertEqual(expected, self.component1.get_license_expression_spdx_id()) + def test_component_model_license_expression_spdx_properties(self): + self.license1.spdx_license_key = "SPDX-1" + self.license1.save() + + expression = "{} AND {}".format(self.license1.key, self.license2.key) + self.component1.license_expression = expression + self.component1.declared_license_expression = expression + self.component1.other_license_expression = expression + self.component1.save() + + expected = "SPDX-1 AND LicenseRef-dejacode-license2" + self.assertEqual(expected, self.component1.concluded_license_expression_spdx) + self.assertEqual(expected, self.component1.declared_license_expression_spdx) + self.assertEqual(expected, self.component1.other_license_expression_spdx) + + self.component1.license_expression = "unknown" + self.component1.save() + expected = "Unknown license key(s): unknown" + self.assertEqual(expected, self.component1.concluded_license_expression_spdx) + + def test_component_model_get_expression_as_spdx(self): + pass + def test_get_license_expression_key_as_link_conflict(self): # self.license1.key is contained in self.license2.key self.license1.key = "w3c" From ae2120b7993acaeeaa6e5922bf37e8affead7974 Mon Sep 17 00:00:00 2001 From: tdruez Date: Wed, 3 Jul 2024 15:43:30 +0400 Subject: [PATCH 26/27] Fix and add unit tests #63 Signed-off-by: tdruez --- component_catalog/license_expression_dje.py | 2 +- .../tests/test_license_expression_dje.py | 26 +++++++++++++++++++ component_catalog/tests/test_models.py | 13 +++++++++- component_catalog/tests/test_views.py | 12 ++++++++- 4 files changed, 50 insertions(+), 3 deletions(-) diff --git a/component_catalog/license_expression_dje.py b/component_catalog/license_expression_dje.py index 6e347314..1085dcf0 100644 --- a/component_catalog/license_expression_dje.py +++ b/component_catalog/license_expression_dje.py @@ -439,4 +439,4 @@ def get_expression_as_spdx(expression, dataspace): """Return an SPDX license expression built from the ``expression``.""" licensing = get_dataspace_licensing(dataspace) parsed_expression = parse_expression(expression, licensing) - return parsed_expression.render(template="{symbol.spdx_license_key}") + return parsed_expression.render(template="{symbol.spdx_id}") diff --git a/component_catalog/tests/test_license_expression_dje.py b/component_catalog/tests/test_license_expression_dje.py index b44ad47c..6c8e6531 100644 --- a/component_catalog/tests/test_license_expression_dje.py +++ b/component_catalog/tests/test_license_expression_dje.py @@ -11,6 +11,7 @@ from collections import namedtuple from itertools import zip_longest +from django.core.cache import caches from django.core.exceptions import ValidationError from django.test import TestCase @@ -20,11 +21,14 @@ from license_expression import as_symbols from component_catalog.license_expression_dje import build_licensing +from component_catalog.license_expression_dje import fetch_licensing_for_dataspace +from component_catalog.license_expression_dje import get_dataspace_licensing from component_catalog.license_expression_dje import get_expression_as_spdx from component_catalog.license_expression_dje import get_license_objects from component_catalog.license_expression_dje import get_unique_license_keys from component_catalog.license_expression_dje import normalize_and_validate_expression from component_catalog.license_expression_dje import parse_expression +from component_catalog.license_expression_dje import render_expression_as_html from dje.models import Dataspace from license_library.models import License from organization.models import Owner @@ -289,6 +293,28 @@ def test_get_expression_as_spdx(self): with self.assertRaises(ExpressionError): get_expression_as_spdx("unknown", self.dataspace) + def test_fetch_licensing_for_dataspace(self): + licensing = fetch_licensing_for_dataspace(self.dataspace) + self.assertEqual([self.license1.key], list(licensing.known_symbols.keys())) + + def test_get_dataspace_licensing(self): + licensing_cache = caches["licensing"] + cache_key = str({self.dataspace.name}) + self.assertFalse(licensing_cache.has_key(cache_key)) + + licensing = get_dataspace_licensing(self.dataspace) + self.assertEqual([self.license1.key], list(licensing.known_symbols.keys())) + self.assertTrue(licensing_cache.has_key(cache_key)) + self.assertTrue(licensing_cache.get(cache_key)) + + def test_render_expression_as_html(self): + expression_as_html = render_expression_as_html(str(self.license1.key), self.dataspace) + expected = 'apache-2.0' + self.assertEqual(expected, expression_as_html) + + expression_as_html = render_expression_as_html("unknown", self.dataspace) + self.assertEqual("unknown", expression_as_html) + def _print_sequence_diff(left, right): for lft, rht in zip_longest(left.split(), right.split()): diff --git a/component_catalog/tests/test_models.py b/component_catalog/tests/test_models.py index e66a14f0..4331c1d1 100644 --- a/component_catalog/tests/test_models.py +++ b/component_catalog/tests/test_models.py @@ -459,12 +459,23 @@ def test_component_model_license_expression_spdx_properties(self): self.assertEqual(expected, self.component1.other_license_expression_spdx) self.component1.license_expression = "unknown" + self.component1.declared_license_expression = "unknown" + self.component1.other_license_expression = "unknown" self.component1.save() expected = "Unknown license key(s): unknown" self.assertEqual(expected, self.component1.concluded_license_expression_spdx) + self.assertEqual(expected, self.component1.declared_license_expression_spdx) + self.assertEqual(expected, self.component1.other_license_expression_spdx) def test_component_model_get_expression_as_spdx(self): - pass + self.license1.spdx_license_key = "SPDX-1" + self.license1.save() + + expression_as_spdx = self.component1.get_expression_as_spdx(str(self.license1.key)) + self.assertEqual("SPDX-1", expression_as_spdx) + + expression_as_spdx = self.component1.get_expression_as_spdx("unknown") + self.assertEqual("Unknown license key(s): unknown", expression_as_spdx) def test_get_license_expression_key_as_link_conflict(self): # self.license1.key is contained in self.license2.key diff --git a/component_catalog/tests/test_views.py b/component_catalog/tests/test_views.py index b90b12c5..6608bb6d 100644 --- a/component_catalog/tests/test_views.py +++ b/component_catalog/tests/test_views.py @@ -378,7 +378,7 @@ def test_component_catalog_detail_view_license_tab(self): # Check the tag set to True is displayed self.assertTrue(self.license_assigned_tag1.value) - # self.assertContains(response, f'{self.license_tag1.label}') + self.assertContains(response, f"{self.license_tag1.label}") self.assertContains(response, f' data-bs-content="{self.license_tag1.text}"') # Check the ordering of the tables respect the license_expression ordering @@ -401,6 +401,16 @@ def no_whitespace(s): expected = "{}{}".format(license2_str, license1_str) self.assertIn(no_whitespace(expected), no_whitespace(response.content)) + def test_component_catalog_detail_view_license_tab_licenses_fields(self): + self.client.login(username="nexb_user", password="t3st") + + self.component1.declared_license_expression = self.license1.key + self.component1.other_license_expression = self.license1.key + self.component1.save() + response = self.client.get(self.component1.get_absolute_url()) + self.assertContains(response, "field-declared-license-expression") + self.assertContains(response, "field-other-license-expression") + def test_return_to_component_from_license_details(self): # Making sure a 'Return to Component' link is available on a License # details view when coming from a Component details view From bc0d44bc4a4b5e251c99c1fe215638948d956d28 Mon Sep 17 00:00:00 2001 From: tdruez Date: Wed, 3 Jul 2024 16:10:01 +0400 Subject: [PATCH 27/27] Add changelog entry #63 Signed-off-by: tdruez --- CHANGELOG.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8d1da3af..ac905e21 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -28,6 +28,29 @@ Release notes - Populate the Package notice_text using "*NOTICE*" file content from Scan "key files". https://github.com/nexB/dejacode/issues/136 +- Added 2 new license related fields on the Component and Package models: + * declared_license_expression + * other_license_expression + https://github.com/nexB/dejacode/issues/63 + +- Added 2 properties on the Component and Package models: + * declared_license_expression_spdx (computed from declared_license_expression) + * other_license_expression_spdx (computed from other_license_expression) + https://github.com/nexB/dejacode/issues/63 + +- Removed 2 fields: Package.declared_license and Component.concluded_license + https://github.com/nexB/dejacode/issues/63 + +- The new license fields are automatically populated from the Package scan + "Update packages automatically from scan". + The new license fields are pre-filled in the Package form when using the + "Add Package" from a PurlDB entry. + The new license fields are pre-filled in the Component form when using the + "Add Component from Package data". + The license expression values provided in the form for the new field is now + properly checked and return a validation error when incorrect. + https://github.com/nexB/dejacode/issues/63 + ### Version 5.1.0 - Upgrade Python version to 3.12 and Django to 5.0.x