Move validate outside of index lock

Signed-off-by: Ayan Sinha Mahapatra <ayansmahapatra@gmail.com>

Author: AyanSinhaMahapatra

etc/scripts/licenses/buildrules.py

@@ -213,7 +213,7 @@ def cli(licenses_file):
         rl = models.BasicRule(text=rdata.text, **rdata.data)
         skinny_rules.append(rl)
 
-    models.validate_rules(skinny_rules, licenses_by_key, with_text=True)
+    models.validate_rules(skinny_rules, licenses_by_key, with_text=True, thorough=True)
 
     print()
     for rule in skinny_rules:

src/licensedcode/cache.py

@@ -130,32 +130,32 @@ def load_or_build(
 
         lock_file = os.path.join(scancode_cache_dir, LICENSE_LOCKFILE_NAME)
 
+        additional_directories = []
+        if only_builtin:
+            additional_directory = None
+            plugin_directories = []
+        else:
+            plugin_directories = get_paths_to_installed_licenses_and_rules()
+            if plugin_directories:
+                additional_directories.extend(plugin_directories)
+
+            # include installed licenses
+            if additional_directory:
+                # additional_directories is originally a tuple
+                additional_directories.append(additional_directory)
+
+        additional_license_dirs = get_license_dirs(additional_dirs=additional_directories)
+        validate_additional_license_data(
+            additional_directories=additional_license_dirs,
+            scancode_license_dir=licenses_data_dir
+        )
+
         # here, we have no cache: lock, check and rebuild
         try:
             # acquire lock and wait until timeout to get a lock or die
             with lockfile.FileLock(lock_file).locked(timeout=timeout):
                 # Here, the cache is either stale or non-existing: we need to
                 # rebuild all cached data (e.g. mostly the index) and cache it
-
-                additional_directories = []
-                if only_builtin:
-                    additional_directory = None
-                    plugin_directories = []
-                else:
-                    plugin_directories = get_paths_to_installed_licenses_and_rules()
-                    if plugin_directories:
-                        additional_directories.extend(plugin_directories)
-
-                    # include installed licenses
-                    if additional_directory:
-                        # additional_directories is originally a tuple
-                        additional_directories.append(additional_directory)
-
-                additional_license_dirs = get_license_dirs(additional_dirs=additional_directories)
-                validate_additional_license_data(
-                    additional_directories=additional_license_dirs,
-                    scancode_license_dir=licenses_data_dir
-                )
                 licenses_db = load_licenses_from_multiple_dirs(
                     additional_license_data_dirs=additional_license_dirs,
                     builtin_license_data_dir=licenses_data_dir,

src/licensedcode/models.py

@@ -837,6 +837,7 @@ def get_rules(
     licenses_data_dir=licenses_data_dir,
     rules_data_dir=rules_data_dir,
     validate=False,
+    validate_thorough=False,
     is_builtin=True,
 ):
     """
@@ -854,7 +855,7 @@ def get_rules(
     ))
 
     if validate:
-        validate_rules(rules=rules, licenses_by_key=licenses_db)
+        validate_rules(rules=rules, licenses_by_key=licenses_db, thorough=validate_thorough)
 
     licenses_as_rules = build_rules_from_licenses(licenses_db)
     return chain(licenses_as_rules, rules)
@@ -951,7 +952,7 @@ def get_rules_from_multiple_dirs(
             is_builtin=False,
         ))
 
-    validate_rules(rules=combined_rules, licenses_by_key=licenses_db)
+    validate_rules(rules=combined_rules, licenses_by_key=licenses_db, thorough=False)
 
     return combined_rules
 
@@ -1029,7 +1030,7 @@ class InvalidLicense(Exception):
     pass
 
 
-def _validate_all_rules(rules, licenses_by_key):
+def _validate_all_rules(rules, licenses_by_key, thorough=False):
     """
     Return a mapping of {error message: [list of Rule]} from validating a list
     of ``rules`` Rule integrity and correctness using known licenses from a
@@ -1039,18 +1040,18 @@ def _validate_all_rules(rules, licenses_by_key):
     errors = defaultdict(list)
 
     for rule in rules:
-        for err_msg in rule.validate(licensing, thorough=True):
+        for err_msg in rule.validate(licensing, thorough=thorough):
             errors[err_msg].append(rule)
     return errors
 
 
-def validate_rules(rules, licenses_by_key, with_text=False, rules_data_dir=rules_data_dir):
+def validate_rules(rules, licenses_by_key, with_text=False, rules_data_dir=rules_data_dir, thorough=False):
     """
     Return a mapping of {error message: [list of Rule]) from validating a list
     of ``rules`` Rule integrity and correctness using known licenses from a
     mapping of ``licenses_by_key`` {key: License}`.
     """
-    errors = _validate_all_rules(rules=rules, licenses_by_key=licenses_by_key)
+    errors = _validate_all_rules(rules=rules, licenses_by_key=licenses_by_key, thorough=thorough)
     if errors:
         message = ['Errors while validating rules:']
         for msg, rules in errors.items():

tests/licensedcode/test_models.py

@@ -187,7 +187,7 @@ class TestRule(FileBasedTesting):
     test_data_dir = TEST_DATA_DIR
 
     def test_validate_license_rules_data(self):
-        rules = list(models.get_rules(validate=True))
+        rules = list(models.get_rules(validate=True, validate_thorough=True))
 
     def test_create_rule_ignore_punctuation(self):
         test_rule = create_rule_from_text_and_expression(text='A one. A two. A three.')