diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
index e50d67710..bc4a04bce 100644
--- a/.github/workflows/docs.yaml
+++ b/.github/workflows/docs.yaml
@@ -64,4 +64,4 @@ jobs:
         git config --global user.email 'not@mail.com'
         git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}
         git commit -am "$last_commit"
-        git push
+        git diff --quiet @{u} HEAD || git push
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index e0ce8ecba..f2543bc53 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -106,8 +106,7 @@ jobs:
       run: |  # we need a separate step because of the bash-specific if-statement in the previous one.
         pytest -n 4 --durations=20 --dist load -sv --reruns 5 --reruns-delay 1
     - name: Check for files left behind by test
-      # skip 3.8 as it fails only for Python 3.8 for no explainable reason.
-      if: matrix.os != 'windows-latest' && matrix.python-version != '3.8' && always()
+      if: matrix.os != 'windows-latest' && always()
       run: |
         before="${{ steps.status-before.outputs.BEFORE }}"
         after="$(git status --porcelain -b)"
diff --git a/tests/conftest.py b/tests/conftest.py
index 62fe3c7e8..81c7c0d5a 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -25,6 +25,7 @@
 
 import logging
 import os
+import shutil
 from pathlib import Path
 import pytest
 
@@ -164,6 +165,15 @@ def pytest_sessionfinish() -> None:
         # Local file deletion
         new_file_list = read_file_list()
         compare_delete_files(file_list, new_file_list)
+
+        # Delete any test dirs that remain
+        # In edge cases due to a mixture of pytest parametrization and oslo concurrency,
+        # some file lock are created after leaving the test. This removes these files!
+        test_files_dir=Path(__file__).parent.parent / "openml"
+        for f in test_files_dir.glob("tests.*"):
+            if f.is_dir():
+                shutil.rmtree(f)
+
         logger.info("Local files deleted")
 
     logger.info(f"{worker} is killed")