Extend inequal join

.clang-tidy

@@ -27,6 +27,8 @@ Checks: [
   '-bugprone-not-null-terminated-result',
   '-bugprone-reserved-identifier', # useful but too slow, TODO retry when https://reviews.llvm.org/rG1c282052624f9d0bd273bde0b47b30c96699c6c7 is merged
   '-bugprone-unchecked-optional-access',
+  '-bugprone-crtp-constructor-accessibility',
+  '-bugprone-suspicious-stringview-data-usage',
 
   '-cert-dcl16-c',
   '-cert-dcl37-c',
@@ -36,6 +38,7 @@ Checks: [
   '-cert-msc51-cpp',
   '-cert-oop54-cpp',
   '-cert-oop57-cpp',
+  '-cert-err33-c', # Misreports on clang-19: it warns about all functions containing 'remove' in the name, not only about the standard library.
 
   '-clang-analyzer-optin.performance.Padding',
 
@@ -99,6 +102,7 @@ Checks: [
   '-modernize-use-emplace',
   '-modernize-use-nodiscard',
   '-modernize-use-trailing-return-type',
+  '-modernize-use-designated-initializers',
 
   '-performance-enum-size',
   '-performance-inefficient-string-concatenation',

.github/actions/check_workflow/action.yml

@@ -0,0 +1,21 @@
+name: CheckWorkflowResults
+
+description: Check overall workflow status and post error to slack if any
+
+inputs:
+  needs:
+    description: github needs context as a json string
+    required: true
+    type: string
+
+runs:
+  using: "composite"
+  steps:
+    - name: Check Workflow
+      shell: bash
+      run: |
+        export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
+        cat > "$WORKFLOW_RESULT_FILE" << 'EOF'
+        ${{ inputs.needs }}
+        EOF
+        python3 ./tests/ci/ci_buddy.py --check-wf-status

.github/workflows/backport_branches.yml

@@ -36,10 +36,6 @@ jobs:
           cd "$GITHUB_WORKSPACE/tests/ci"
           echo "Testing the main ci directory"
           python3 -m unittest discover -s . -p 'test_*.py'
-          for dir in *_lambda/; do
-            echo "Testing $dir"
-            python3 -m unittest discover -s "$dir" -p 'test_*.py'
-          done
       - name: PrepareRunConfig
         id: runconfig
         run: |
@@ -245,8 +241,9 @@ jobs:
       runner_type: stress-tester
       data: ${{ needs.RunConfig.outputs.data }}
   FinishCheck:
-    if: ${{ !failure() && !cancelled() }}
+    if: ${{ !cancelled() }}
     needs:
+      - RunConfig
       - Builds_Report
       - FunctionalStatelessTestAsan
       - FunctionalStatefulTestDebug
@@ -261,10 +258,23 @@ jobs:
         with:
           clear-repository: true
       - name: Finish label
+        if: ${{ !failure() }}
         run: |
+          export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
+          cat > "$WORKFLOW_RESULT_FILE" << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
           cd "$GITHUB_WORKSPACE/tests/ci"
           # update mergeable check
-          python3 merge_pr.py --set-ci-status --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
+          python3 merge_pr.py --set-ci-status
           # update overall ci report
           python3 finish_check.py --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
           python3 merge_pr.py
+      - name: Check Workflow results
+        if: ${{ !cancelled() }}
+        run: |
+          export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
+          cat > "$WORKFLOW_RESULT_FILE" << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
+          python3 ./tests/ci/ci_buddy.py --check-wf-status

.github/workflows/create_release.yml

@@ -2,7 +2,6 @@ name: CreateRelease
 
 concurrency:
   group: release
-
 'on':
   workflow_dispatch:
     inputs:
@@ -17,10 +16,15 @@ concurrency:
         options:
           - patch
           - new
+      only-repo:
+        description: 'Run only repos updates including docker (repo-recovery, tests)'
+        required: false
+        default: false
+        type: boolean
       dry-run:
         description: 'Dry run'
         required: false
-        default: true
+        default: false
         type: boolean
 
 jobs:
@@ -31,78 +35,88 @@ jobs:
     steps:
       - name: DebugInfo
         uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6
-      - name: Set envs
-        # https://docs.github.com/en/actions/learn-github-actions/workflow-commands-for-github-actions#multiline-strings
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
-          ${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
-          RCSK
-          RELEASE_INFO_FILE=${{ runner.temp }}/release_info.json
-          EOF
       - name: Check out repository code
         uses: ClickHouse/checkout@v1
         with:
           token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
           fetch-depth: 0
       - name: Prepare Release Info
+        shell: bash
         run: |
+          if [ ${{ inputs.only-repo }} == "true" ]; then
+              git tag -l ${{ inputs.ref }} || { echo "With only-repo option ref must be a valid release tag"; exit 1; }
+          fi
           python3 ./tests/ci/create_release.py --prepare-release-info \
             --ref ${{ inputs.ref }} --release-type ${{ inputs.type }} \
-            --outfile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }}
+            ${{ inputs.dry-run == true && '--dry-run' || '' }} \
+            ${{ inputs.only-repo == true && '--skip-tag-check' || '' }}
           echo "::group::Release Info"
-          python3 -m json.tool "$RELEASE_INFO_FILE"
+          python3 -m json.tool /tmp/release_info.json
           echo "::endgroup::"
-          release_tag=$(jq -r '.release_tag' "$RELEASE_INFO_FILE")
-          commit_sha=$(jq -r '.commit_sha' "$RELEASE_INFO_FILE")
+          release_tag=$(jq -r '.release_tag' /tmp/release_info.json)
+          commit_sha=$(jq -r '.commit_sha' /tmp/release_info.json)
+          is_latest=$(jq -r '.latest' /tmp/release_info.json)
           echo "Release Tag: $release_tag"
           echo "RELEASE_TAG=$release_tag" >> "$GITHUB_ENV"
           echo "COMMIT_SHA=$commit_sha" >> "$GITHUB_ENV"
+          if [ "$is_latest" == "true" ]; then
+            echo "DOCKER_TAG_TYPE=release-latest" >> "$GITHUB_ENV"
+          else
+            echo "DOCKER_TAG_TYPE=release" >> "$GITHUB_ENV"
+          fi
       - name: Download All Release Artifacts
         if: ${{ inputs.type == 'patch' }}
+        shell: bash
         run: |
-          python3 ./tests/ci/create_release.py --infile "$RELEASE_INFO_FILE" --download-packages ${{ inputs.dry-run && '--dry-run' || '' }}
+          python3 ./tests/ci/create_release.py --download-packages ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Push Git Tag for the Release
+        if: ${{ ! inputs.only-repo }}
+        shell: bash
         run: |
-          python3 ./tests/ci/create_release.py --push-release-tag --infile "$RELEASE_INFO_FILE" ${{ inputs.dry-run && '--dry-run' || '' }}
+          python3 ./tests/ci/create_release.py --push-release-tag ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Push New Release Branch
-        if: ${{ inputs.type == 'new' }}
+        if: ${{ inputs.type == 'new' && ! inputs.only-repo }}
+        shell: bash
         run: |
-          python3 ./tests/ci/create_release.py --push-new-release-branch --infile "$RELEASE_INFO_FILE" ${{ inputs.dry-run && '--dry-run' || '' }}
+          python3 ./tests/ci/create_release.py --push-new-release-branch ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Bump CH Version and Update Contributors' List
+        if: ${{ ! inputs.only-repo }}
+        shell: bash
         run: |
-          python3 ./tests/ci/create_release.py --create-bump-version-pr --infile "$RELEASE_INFO_FILE"  ${{ inputs.dry-run && '--dry-run' || '' }}
-      - name: Checkout master
-        run: |
-          git checkout master
+          python3 ./tests/ci/create_release.py --create-bump-version-pr ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Bump Docker versions, Changelog, Security
-        if: ${{ inputs.type == 'patch' }}
+        if: ${{ inputs.type == 'patch' && ! inputs.only-repo }}
+        shell: bash
         run: |
-          [ "$(git branch --show-current)" != "master" ] && echo "not on the master" && exit 1
+          python3 ./tests/ci/create_release.py --set-progress-started --progress "update changelog, docker version, security"
+
+          git checkout master  # in case WF started from feature branch
           echo "List versions"
           ./utils/list-versions/list-versions.sh > ./utils/list-versions/version_date.tsv
           echo "Update docker version"
           ./utils/list-versions/update-docker-version.sh
           echo "Generate ChangeLog"
           export CI=1
           docker run -u "${UID}:${GID}" -e PYTHONUNBUFFERED=1 -e CI=1 --network=host \
-            --volume=".:/ClickHouse" clickhouse/style-test \
-            /ClickHouse/tests/ci/changelog.py -v --debug-helpers \
-            --gh-user-or-token="$GH_TOKEN" --jobs=5 \
-            --output="/ClickHouse/docs/changelogs/${{ env.RELEASE_TAG }}.md" ${{ env.RELEASE_TAG }}
+            --volume=".:/wd" --workdir="/wd" \
+            clickhouse/style-test \
+            ./tests/ci/changelog.py -v --debug-helpers \
+            --jobs=5 \
+            --output="./docs/changelogs/${{ env.RELEASE_TAG }}.md" ${{ env.RELEASE_TAG }}
           git add ./docs/changelogs/${{ env.RELEASE_TAG }}.md
           echo "Generate Security"
           python3 ./utils/security-generator/generate_security.py > SECURITY.md
           git diff HEAD
       - name: Create ChangeLog PR
-        if: ${{ inputs.type == 'patch' && ! inputs.dry-run }}
+        if: ${{ inputs.type == 'patch' && ! inputs.dry-run && ! inputs.only-repo }}
         uses: peter-evans/create-pull-request@v6
         with:
           author: "robot-clickhouse <robot-clickhouse@users.noreply.github.com>"
           token: ${{ secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN }}
           committer: "robot-clickhouse <robot-clickhouse@users.noreply.github.com>"
           commit-message: Update version_date.tsv and changelogs after ${{ env.RELEASE_TAG }}
           branch: auto/${{ env.RELEASE_TAG }}
+          base: master
           assignees: ${{ github.event.sender.login }}  # assign the PR to the tag pusher
           delete-branch: true
           title: Update version_date.tsv and changelog after ${{ env.RELEASE_TAG }}
@@ -111,56 +125,78 @@ jobs:
             Update version_date.tsv and changelogs after ${{ env.RELEASE_TAG }}
             ### Changelog category (leave one):
             - Not for changelog (changelog entry is not required)
-      - name: Reset changes if Dry-run
-        if: ${{ inputs.dry-run }}
+      - name: Complete previous steps and Restore git state
+        if: ${{ inputs.type == 'patch' && ! inputs.only-repo }}
+        shell: bash
         run: |
+          python3 ./tests/ci/create_release.py --set-progress-completed
           git reset --hard HEAD
-      - name: Checkout back to GITHUB_REF
-        run: |
           git checkout "$GITHUB_REF_NAME"
       - name: Create GH Release
-        if: ${{ inputs.type == 'patch' }}
+        if: ${{ inputs.type == 'patch' && ! inputs.only-repo }}
+        shell: bash
         run: |
-          python3 ./tests/ci/create_release.py --create-gh-release \
-            --infile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }}
-
+          python3 ./tests/ci/create_release.py --create-gh-release ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Export TGZ Packages
         if: ${{ inputs.type == 'patch' }}
+        shell: bash
         run: |
-          python3 ./tests/ci/artifactory.py --export-tgz --infile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }}
+          python3 ./tests/ci/artifactory.py --export-tgz ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Test TGZ Packages
         if: ${{ inputs.type == 'patch' }}
+        shell: bash
         run: |
-          python3 ./tests/ci/artifactory.py --test-tgz --infile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }}
+          python3 ./tests/ci/artifactory.py --test-tgz ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Export RPM Packages
         if: ${{ inputs.type == 'patch' }}
+        shell: bash
         run: |
-          python3 ./tests/ci/artifactory.py --export-rpm --infile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }}
+          python3 ./tests/ci/artifactory.py --export-rpm ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Test RPM Packages
         if: ${{ inputs.type == 'patch' }}
+        shell: bash
         run: |
-          python3 ./tests/ci/artifactory.py --test-rpm --infile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }}
+          python3 ./tests/ci/artifactory.py --test-rpm ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Export Debian Packages
         if: ${{ inputs.type == 'patch' }}
+        shell: bash
         run: |
-            python3 ./tests/ci/artifactory.py --export-debian --infile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }}
+          python3 ./tests/ci/artifactory.py --export-debian ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Test Debian Packages
         if: ${{ inputs.type == 'patch' }}
+        shell: bash
         run: |
-          python3 ./tests/ci/artifactory.py --test-debian --infile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }}
+          python3 ./tests/ci/artifactory.py --test-debian ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Docker clickhouse/clickhouse-server building
         if: ${{ inputs.type == 'patch' }}
+        shell: bash
         run: |
           cd "./tests/ci"
+          python3 ./create_release.py --set-progress-started --progress "docker server release"
           export CHECK_NAME="Docker server image"
-          python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push' || '' }}
+          python3 docker_server.py --tag-type ${{ env.DOCKER_TAG_TYPE }} --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push' || '' }}
+          python3 ./create_release.py --set-progress-completed
       - name: Docker clickhouse/clickhouse-keeper building
         if: ${{ inputs.type == 'patch' }}
+        shell: bash
         run: |
           cd "./tests/ci"
+          python3 ./create_release.py --set-progress-started --progress "docker keeper release"
           export CHECK_NAME="Docker keeper image"
-          python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push'  || '' }}
+          python3 docker_server.py --tag-type ${{ env.DOCKER_TAG_TYPE }} --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push'  || '' }}
+          python3 ./create_release.py --set-progress-completed
+      - name: Update release info. Merge created PRs
+        shell: bash
+        run: |
+          python3 ./tests/ci/create_release.py --merge-prs ${{ inputs.dry-run == true && '--dry-run' || '' }}
+      - name: Set current Release progress to Completed with OK
+        shell: bash
+        run: |
+          # dummy stage to finalize release info with "progress: completed; status: OK"
+          python3 ./tests/ci/create_release.py --set-progress-started --progress "completed"
+          python3 ./tests/ci/create_release.py --set-progress-completed
       - name: Post Slack Message
-        if: always()
+        if: ${{ !cancelled() }}
+        shell: bash
         run: |
-          echo Slack Message
+          python3 ./tests/ci/create_release.py --post-status ${{ inputs.dry-run == true && '--dry-run' || '' }}