diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 0000000000..5cbc2d3f5f --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,4 @@ +# do not notify until at least 100 builds have been uploaded from the CI pipeline +# you can also set after_n_builds on comments independently +comment: + after_n_builds: 100 diff --git a/.evergreen/generated_configs/functions.yml b/.evergreen/generated_configs/functions.yml index 58bffbf922..9fbc1f8500 100644 --- a/.evergreen/generated_configs/functions.yml +++ b/.evergreen/generated_configs/functions.yml @@ -250,6 +250,7 @@ functions: working_dir: src include_expansions_in_env: - TOOLCHAIN_VERSION + - COVERAGE type: test # Upload coverage codecov @@ -268,7 +269,7 @@ functions: - github_pr_number - github_pr_head_branch - github_author - - is_patch + - requester - branch_name type: test diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index 60ee6ed135..6a460ce486 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -75,7 +75,7 @@ tasks: SUB_TEST_NAME: session-creds TOOLCHAIN_VERSION: 3.14t tags: [auth-aws, auth-aws-session-creds, free-threaded] - - name: test-auth-aws-rapid-web-identity-python3.14 + - name: test-auth-aws-rapid-web-identity-python3.14-cov commands: - func: run server vars: @@ -87,7 +87,8 @@ tasks: TEST_NAME: auth_aws SUB_TEST_NAME: web-identity TOOLCHAIN_VERSION: "3.14" - tags: [auth-aws, auth-aws-web-identity] + COVERAGE: "1" + tags: [auth-aws, auth-aws-web-identity, pr] - name: test-auth-aws-rapid-web-identity-session-name-python3.14 commands: - func: run server @@ -904,7 +905,7 @@ tasks: - ocsp-ecdsa - rapid - ocsp-staple - - name: test-ocsp-ecdsa-valid-cert-server-staples-latest-python3.14 + - name: test-ocsp-ecdsa-valid-cert-server-staples-latest-python3.14-cov commands: - func: run tests vars: @@ -913,11 +914,13 @@ tasks: TEST_NAME: ocsp TOOLCHAIN_VERSION: "3.14" VERSION: latest + COVERAGE: "1" tags: - ocsp - ocsp-ecdsa - latest - ocsp-staple + - pr - name: test-ocsp-ecdsa-invalid-cert-server-staples-v4.4-python3.10-min-deps commands: - func: run tests @@ -1928,7 +1931,7 @@ tasks: - ocsp-rsa - rapid - ocsp-staple - - name: test-ocsp-rsa-valid-cert-server-staples-latest-python3.14 + - name: test-ocsp-rsa-valid-cert-server-staples-latest-python3.14-cov commands: - func: run tests vars: @@ -1937,11 +1940,13 @@ tasks: TEST_NAME: ocsp TOOLCHAIN_VERSION: "3.14" VERSION: latest + COVERAGE: "1" tags: - ocsp - ocsp-rsa - latest - ocsp-staple + - pr - name: test-ocsp-rsa-invalid-cert-server-staples-v4.4-python3.10-min-deps commands: - func: run tests @@ -2615,20 +2620,18 @@ tasks: - replica_set-auth-nossl - async - free-threaded - - name: test-server-version-python3.13-sync-auth-nossl-replica-set-cov + - name: test-server-version-python3.13-sync-auth-nossl-replica-set commands: - func: run server vars: AUTH: auth SSL: nossl TOPOLOGY: replica_set - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: nossl TOPOLOGY: replica_set - COVERAGE: "1" TOOLCHAIN_VERSION: "3.13" TEST_NAME: default_sync tags: @@ -2636,20 +2639,18 @@ tasks: - python-3.13 - replica_set-auth-nossl - sync - - name: test-server-version-python3.12-async-auth-ssl-replica-set-cov + - name: test-server-version-python3.12-async-auth-ssl-replica-set commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: replica_set - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: replica_set - COVERAGE: "1" TOOLCHAIN_VERSION: "3.12" TEST_NAME: default_async tags: @@ -2657,20 +2658,18 @@ tasks: - python-3.12 - replica_set-auth-ssl - async - - name: test-server-version-python3.11-sync-auth-ssl-replica-set-cov + - name: test-server-version-python3.11-sync-auth-ssl-replica-set commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: replica_set - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: replica_set - COVERAGE: "1" TOOLCHAIN_VERSION: "3.11" TEST_NAME: default_sync tags: @@ -2743,20 +2742,18 @@ tasks: - python-pypy3.11 - replica_set-noauth-ssl - async - - name: test-server-version-python3.14-sync-noauth-ssl-replica-set-cov + - name: test-server-version-python3.14-sync-noauth-ssl-replica-set commands: - func: run server vars: AUTH: noauth SSL: ssl TOPOLOGY: replica_set - COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: ssl TOPOLOGY: replica_set - COVERAGE: "1" TOOLCHAIN_VERSION: "3.14" TEST_NAME: default_sync tags: @@ -2764,20 +2761,18 @@ tasks: - python-3.14 - replica_set-noauth-ssl - sync - - name: test-server-version-python3.14-async-auth-nossl-sharded-cluster-cov + - name: test-server-version-python3.14-async-auth-nossl-sharded-cluster commands: - func: run server vars: AUTH: auth SSL: nossl TOPOLOGY: sharded_cluster - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: nossl TOPOLOGY: sharded_cluster - COVERAGE: "1" TOOLCHAIN_VERSION: "3.14" TEST_NAME: default_async tags: @@ -2829,20 +2824,18 @@ tasks: - sharded_cluster-auth-ssl - async - pr - - name: test-server-version-python3.11-async-auth-ssl-sharded-cluster-cov + - name: test-server-version-python3.11-async-auth-ssl-sharded-cluster commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" TOOLCHAIN_VERSION: "3.11" TEST_NAME: default_async tags: @@ -2850,20 +2843,18 @@ tasks: - python-3.11 - sharded_cluster-auth-ssl - async - - name: test-server-version-python3.12-async-auth-ssl-sharded-cluster-cov + - name: test-server-version-python3.12-async-auth-ssl-sharded-cluster commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" TOOLCHAIN_VERSION: "3.12" TEST_NAME: default_async tags: @@ -2871,20 +2862,18 @@ tasks: - python-3.12 - sharded_cluster-auth-ssl - async - - name: test-server-version-python3.13-async-auth-ssl-sharded-cluster-cov + - name: test-server-version-python3.13-async-auth-ssl-sharded-cluster commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" TOOLCHAIN_VERSION: "3.13" TEST_NAME: default_async tags: @@ -2892,20 +2881,18 @@ tasks: - python-3.13 - sharded_cluster-auth-ssl - async - - name: test-server-version-python3.14-async-auth-ssl-sharded-cluster-cov + - name: test-server-version-python3.14-async-auth-ssl-sharded-cluster commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" TOOLCHAIN_VERSION: "3.14" TEST_NAME: default_async tags: @@ -2976,20 +2963,18 @@ tasks: - sharded_cluster-auth-ssl - sync - pr - - name: test-server-version-python3.11-sync-auth-ssl-sharded-cluster-cov + - name: test-server-version-python3.11-sync-auth-ssl-sharded-cluster commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" TOOLCHAIN_VERSION: "3.11" TEST_NAME: default_sync tags: @@ -2997,20 +2982,18 @@ tasks: - python-3.11 - sharded_cluster-auth-ssl - sync - - name: test-server-version-python3.12-sync-auth-ssl-sharded-cluster-cov + - name: test-server-version-python3.12-sync-auth-ssl-sharded-cluster commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" TOOLCHAIN_VERSION: "3.12" TEST_NAME: default_sync tags: @@ -3018,20 +3001,18 @@ tasks: - python-3.12 - sharded_cluster-auth-ssl - sync - - name: test-server-version-python3.13-sync-auth-ssl-sharded-cluster-cov + - name: test-server-version-python3.13-sync-auth-ssl-sharded-cluster commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" TOOLCHAIN_VERSION: "3.13" TEST_NAME: default_sync tags: @@ -3039,20 +3020,18 @@ tasks: - python-3.13 - sharded_cluster-auth-ssl - sync - - name: test-server-version-python3.14-sync-auth-ssl-sharded-cluster-cov + - name: test-server-version-python3.14-sync-auth-ssl-sharded-cluster commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster - COVERAGE: "1" TOOLCHAIN_VERSION: "3.14" TEST_NAME: default_sync tags: @@ -3099,20 +3078,18 @@ tasks: - python-pypy3.11 - sharded_cluster-auth-ssl - sync - - name: test-server-version-python3.12-async-noauth-nossl-sharded-cluster-cov + - name: test-server-version-python3.12-async-noauth-nossl-sharded-cluster commands: - func: run server vars: AUTH: noauth SSL: nossl TOPOLOGY: sharded_cluster - COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: nossl TOPOLOGY: sharded_cluster - COVERAGE: "1" TOOLCHAIN_VERSION: "3.12" TEST_NAME: default_async tags: @@ -3120,20 +3097,18 @@ tasks: - python-3.12 - sharded_cluster-noauth-nossl - async - - name: test-server-version-python3.11-sync-noauth-nossl-sharded-cluster-cov + - name: test-server-version-python3.11-sync-noauth-nossl-sharded-cluster commands: - func: run server vars: AUTH: noauth SSL: nossl TOPOLOGY: sharded_cluster - COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: nossl TOPOLOGY: sharded_cluster - COVERAGE: "1" TOOLCHAIN_VERSION: "3.11" TEST_NAME: default_sync tags: @@ -3141,7 +3116,7 @@ tasks: - python-3.11 - sharded_cluster-noauth-nossl - sync - - name: test-server-version-python3.10-async-noauth-ssl-sharded-cluster-min-deps-cov + - name: test-server-version-python3.10-async-noauth-ssl-sharded-cluster-min-deps commands: - func: run server vars: @@ -3149,14 +3124,12 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster TEST_MIN_DEPS: "1" - COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: ssl TOPOLOGY: sharded_cluster TEST_MIN_DEPS: "1" - COVERAGE: "1" TOOLCHAIN_VERSION: "3.10" TEST_NAME: default_async tags: @@ -3183,20 +3156,18 @@ tasks: - python-pypy3.11 - sharded_cluster-noauth-ssl - sync - - name: test-server-version-python3.13-async-auth-nossl-standalone-cov + - name: test-server-version-python3.13-async-auth-nossl-standalone commands: - func: run server vars: AUTH: auth SSL: nossl TOPOLOGY: standalone - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: nossl TOPOLOGY: standalone - COVERAGE: "1" TOOLCHAIN_VERSION: "3.13" TEST_NAME: default_async tags: @@ -3204,20 +3175,18 @@ tasks: - python-3.13 - standalone-auth-nossl - async - - name: test-server-version-python3.12-sync-auth-nossl-standalone-cov + - name: test-server-version-python3.12-sync-auth-nossl-standalone commands: - func: run server vars: AUTH: auth SSL: nossl TOPOLOGY: standalone - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: nossl TOPOLOGY: standalone - COVERAGE: "1" TOOLCHAIN_VERSION: "3.12" TEST_NAME: default_sync tags: @@ -3225,20 +3194,18 @@ tasks: - python-3.12 - standalone-auth-nossl - sync - - name: test-server-version-python3.11-async-auth-ssl-standalone-cov + - name: test-server-version-python3.11-async-auth-ssl-standalone commands: - func: run server vars: AUTH: auth SSL: ssl TOPOLOGY: standalone - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: standalone - COVERAGE: "1" TOOLCHAIN_VERSION: "3.11" TEST_NAME: default_async tags: @@ -3246,7 +3213,7 @@ tasks: - python-3.11 - standalone-auth-ssl - async - - name: test-server-version-python3.10-sync-auth-ssl-standalone-min-deps-cov + - name: test-server-version-python3.10-sync-auth-ssl-standalone-min-deps commands: - func: run server vars: @@ -3254,14 +3221,12 @@ tasks: SSL: ssl TOPOLOGY: standalone TEST_MIN_DEPS: "1" - COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: standalone TEST_MIN_DEPS: "1" - COVERAGE: "1" TOOLCHAIN_VERSION: "3.10" TEST_NAME: default_sync tags: @@ -3293,18 +3258,20 @@ tasks: - standalone-noauth-nossl - async - pr - - name: test-server-version-pypy3.11-sync-noauth-nossl-standalone + - name: test-server-version-pypy3.11-sync-noauth-nossl-standalone-cov commands: - func: run server vars: AUTH: noauth SSL: nossl TOPOLOGY: standalone + COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: nossl TOPOLOGY: standalone + COVERAGE: "1" TOOLCHAIN_VERSION: pypy3.11 TEST_NAME: default_sync tags: @@ -3313,20 +3280,18 @@ tasks: - standalone-noauth-nossl - sync - pr - - name: test-server-version-python3.14-async-noauth-ssl-standalone-cov + - name: test-server-version-python3.14-async-noauth-ssl-standalone commands: - func: run server vars: AUTH: noauth SSL: ssl TOPOLOGY: standalone - COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: ssl TOPOLOGY: standalone - COVERAGE: "1" TOOLCHAIN_VERSION: "3.14" TEST_NAME: default_async tags: @@ -4082,7 +4047,7 @@ tasks: - standalone-noauth-nossl - async - pypy - - name: test-standard-latest-python3.12-async-noauth-ssl-replica-set + - name: test-standard-latest-python3.12-async-noauth-ssl-replica-set-cov commands: - func: run server vars: @@ -4090,12 +4055,14 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: latest + COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: ssl TOPOLOGY: replica_set VERSION: latest + COVERAGE: "1" TOOLCHAIN_VERSION: "3.12" TEST_NAME: default_async tags: @@ -4128,7 +4095,7 @@ tasks: - replica_set-noauth-ssl - async - pypy - - name: test-standard-latest-python3.13-async-auth-ssl-sharded-cluster + - name: test-standard-latest-python3.13-async-auth-ssl-sharded-cluster-cov commands: - func: run server vars: @@ -4136,12 +4103,14 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: latest + COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster VERSION: latest + COVERAGE: "1" TOOLCHAIN_VERSION: "3.13" TEST_NAME: default_async tags: @@ -4151,7 +4120,7 @@ tasks: - sharded_cluster-auth-ssl - async - pr - - name: test-standard-latest-python3.11-async-noauth-nossl-standalone + - name: test-standard-latest-python3.11-async-noauth-nossl-standalone-cov commands: - func: run server vars: @@ -4159,12 +4128,14 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: latest + COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: nossl TOPOLOGY: standalone VERSION: latest + COVERAGE: "1" TOOLCHAIN_VERSION: "3.11" TEST_NAME: default_async tags: @@ -4174,7 +4145,7 @@ tasks: - standalone-noauth-nossl - async - pr - - name: test-standard-latest-python3.14-async-noauth-nossl-standalone + - name: test-standard-latest-python3.14-async-noauth-nossl-standalone-cov commands: - func: run server vars: @@ -4182,12 +4153,14 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: latest + COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: nossl TOPOLOGY: standalone VERSION: latest + COVERAGE: "1" TOOLCHAIN_VERSION: "3.14" TEST_NAME: default_async tags: @@ -4829,7 +4802,7 @@ tasks: - python-3.13 - standalone-noauth-nossl - noauth - - name: test-non-standard-latest-python3.14t-noauth-ssl-replica-set + - name: test-non-standard-latest-python3.14t-noauth-ssl-replica-set-cov commands: - func: run server vars: @@ -4837,12 +4810,14 @@ tasks: SSL: ssl TOPOLOGY: replica_set VERSION: latest + COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: ssl TOPOLOGY: replica_set VERSION: latest + COVERAGE: "1" TOOLCHAIN_VERSION: 3.14t tags: - test-non-standard @@ -4874,7 +4849,7 @@ tasks: - replica_set-noauth-ssl - noauth - pypy - - name: test-non-standard-latest-python3.14-auth-ssl-sharded-cluster + - name: test-non-standard-latest-python3.14-auth-ssl-sharded-cluster-cov commands: - func: run server vars: @@ -4882,12 +4857,14 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: latest + COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster VERSION: latest + COVERAGE: "1" TOOLCHAIN_VERSION: "3.14" tags: - test-non-standard @@ -4896,7 +4873,7 @@ tasks: - sharded_cluster-auth-ssl - auth - pr - - name: test-non-standard-latest-python3.13-noauth-nossl-standalone + - name: test-non-standard-latest-python3.13-noauth-nossl-standalone-cov commands: - func: run server vars: @@ -4904,12 +4881,14 @@ tasks: SSL: nossl TOPOLOGY: standalone VERSION: latest + COVERAGE: "1" - func: run tests vars: AUTH: noauth SSL: nossl TOPOLOGY: standalone VERSION: latest + COVERAGE: "1" TOOLCHAIN_VERSION: "3.13" tags: - test-non-standard @@ -5007,7 +4986,7 @@ tasks: - pypy # Test numpy tests - - name: test-numpy-python3.10 + - name: test-numpy-python3.10-python3.10 commands: - func: test numpy vars: @@ -5017,16 +4996,18 @@ tasks: - vector - python-3.10 - test-numpy - - name: test-numpy-python3.14 + - name: test-numpy-python3.14-python3.14-cov commands: - func: test numpy vars: TOOLCHAIN_VERSION: "3.14" + COVERAGE: "1" tags: - binary - vector - python-3.14 - test-numpy + - pr # Test standard auth tests - name: test-standard-auth-v4.2-python3.10-auth-ssl-sharded-cluster-min-deps @@ -5290,7 +5271,7 @@ tasks: - sharded_cluster-auth-ssl - auth - pypy - - name: test-standard-auth-latest-python3.11-auth-ssl-sharded-cluster + - name: test-standard-auth-latest-python3.11-auth-ssl-sharded-cluster-cov commands: - func: run server vars: @@ -5298,12 +5279,14 @@ tasks: SSL: ssl TOPOLOGY: sharded_cluster VERSION: latest + COVERAGE: "1" - func: run tests vars: AUTH: auth SSL: ssl TOPOLOGY: sharded_cluster VERSION: latest + COVERAGE: "1" TOOLCHAIN_VERSION: "3.11" tags: - test-standard-auth diff --git a/.evergreen/generated_configs/variants.yml b/.evergreen/generated_configs/variants.yml index edca050240..4c9116d628 100644 --- a/.evergreen/generated_configs/variants.yml +++ b/.evergreen/generated_configs/variants.yml @@ -368,7 +368,6 @@ buildvariants: run_on: - rhel87-small expansions: - COVERAGE: "1" NO_EXT: "1" # No server tests @@ -420,6 +419,8 @@ buildvariants: run_on: - ubuntu2204-small batchtime: 1440 + expansions: + COVERAGE: "1" tags: [pr] - name: auth-oidc-macos tasks: diff --git a/.evergreen/scripts/generate_config.py b/.evergreen/scripts/generate_config.py index 3375b9e14e..b688a1cd4a 100644 --- a/.evergreen/scripts/generate_config.py +++ b/.evergreen/scripts/generate_config.py @@ -318,7 +318,7 @@ def create_green_framework_variants(): def create_no_c_ext_variants(): host = DEFAULT_HOST tasks = [".test-standard"] - expansions = dict(COVERAGE="1") + expansions = dict() handle_c_ext(C_EXTS[0], expansions) display_name = get_variant_name("No C Ext", host) return [create_variant(tasks, display_name, host=host, expansions=expansions)] @@ -344,8 +344,12 @@ def create_test_numpy_tasks(): tasks = [] for python in MIN_MAX_PYTHON: tags = ["binary", "vector", f"python-{python}", "test-numpy"] - task_name = get_task_name("test-numpy", python=python) - test_func = FunctionCall(func="test numpy", vars=dict(TOOLCHAIN_VERSION=python)) + vars = vars = dict(TOOLCHAIN_VERSION=python) + if python == MIN_MAX_PYTHON[-1]: + tags.append("pr") + vars["COVERAGE"] = "1" + task_name = get_task_name("test-numpy", python=python, **vars) + test_func = FunctionCall(func="test numpy", vars=vars) tasks.append(EvgTask(name=task_name, tags=tags, commands=[test_func])) return tasks @@ -397,6 +401,7 @@ def create_oidc_auth_variants(): tags=["pr"], host=host, batchtime=BATCHTIME_DAY, + expansions=dict(COVERAGE="1"), ) ) return variants @@ -596,7 +601,7 @@ def create_server_version_tasks(): expansions["TEST_MIN_DEPS"] = "1" if "t" in python: tags.append("free-threaded") - if python not in PYPYS and "t" not in python: + if "pr" in tags: expansions["COVERAGE"] = "1" name = get_task_name( "test-server-version", @@ -661,6 +666,8 @@ def create_test_non_standard_tasks(): expansions = dict(AUTH=auth, SSL=ssl, TOPOLOGY=topology, VERSION=version) if python == ALL_PYTHONS[0]: expansions["TEST_MIN_DEPS"] = "1" + elif pr: + expansions["COVERAGE"] = "1" name = get_task_name("test-non-standard", python=python, **expansions) server_func = FunctionCall(func="run server", vars=expansions) test_vars = expansions.copy() @@ -703,6 +710,8 @@ def create_test_standard_auth_tasks(): expansions = dict(AUTH=auth, SSL=ssl, TOPOLOGY=topology, VERSION=version) if python == ALL_PYTHONS[0]: expansions["TEST_MIN_DEPS"] = "1" + elif pr: + expansions["COVERAGE"] = "1" name = get_task_name("test-standard-auth", python=python, **expansions) server_func = FunctionCall(func="run server", vars=expansions) test_vars = expansions.copy() @@ -741,6 +750,8 @@ def create_standard_tasks(): expansions = dict(AUTH=auth, SSL=ssl, TOPOLOGY=topology, VERSION=version) if python == ALL_PYTHONS[0]: expansions["TEST_MIN_DEPS"] = "1" + elif pr: + expansions["COVERAGE"] = "1" name = get_task_name("test-standard", python=python, sync=sync, **expansions) server_func = FunctionCall(func="run server", vars=expansions) test_vars = expansions.copy() @@ -810,8 +821,11 @@ def create_aws_tasks(): if "t" in python: tags.append("free-threaded") test_vars = dict(TEST_NAME="auth_aws", SUB_TEST_NAME=test_type, TOOLCHAIN_VERSION=python) - if python == ALL_PYTHONS[0]: + if python == MIN_MAX_PYTHON[0]: test_vars["TEST_MIN_DEPS"] = "1" + elif python == MIN_MAX_PYTHON[-1]: + tags.append("pr") + test_vars["COVERAGE"] = "1" name = get_task_name(f"{base_name}-{test_type}", **test_vars) test_func = FunctionCall(func="run tests", vars=test_vars) funcs = [server_func, assume_func, test_func] @@ -849,11 +863,11 @@ def create_oidc_tasks(): tasks = [] for sub_test in ["default", "azure", "gcp", "eks", "aks", "gke"]: vars = dict(TEST_NAME="auth_oidc", SUB_TEST_NAME=sub_test) - test_func = FunctionCall(func="run tests", vars=vars) - task_name = f"test-auth-oidc-{sub_test}" tags = ["auth_oidc"] if sub_test != "default": tags.append("auth_oidc_remote") + test_func = FunctionCall(func="run tests", vars=vars) + task_name = get_task_name(f"test-auth-oidc-{sub_test}", **vars) tasks.append(EvgTask(name=task_name, tags=tags, commands=[test_func])) return tasks @@ -903,14 +917,14 @@ def _create_ocsp_tasks(algo, variant, server_type, base_task_name): ) if python == ALL_PYTHONS[0]: vars["TEST_MIN_DEPS"] = "1" - test_func = FunctionCall(func="run tests", vars=vars) - tags = ["ocsp", f"ocsp-{algo}", version] if "disableStapling" not in variant: tags.append("ocsp-staple") - if algo == "valid-cert-server-staples" and version == "latest": + if base_task_name == "valid-cert-server-staples" and version == "latest": tags.append("pr") - + if "TEST_MIN_DEPS" not in vars: + vars["COVERAGE"] = "1" + test_func = FunctionCall(func="run tests", vars=vars) task_name = get_task_name(f"test-ocsp-{algo}-{base_task_name}", **vars) tasks.append(EvgTask(name=task_name, tags=tags, commands=[test_func])) @@ -1087,7 +1101,7 @@ def create_upload_coverage_codecov_func(): "github_pr_number", "github_pr_head_branch", "github_author", - "is_patch", + "requester", "branch_name", ] args = [ @@ -1230,7 +1244,7 @@ def create_run_tests_func(): def create_test_numpy_func(): - includes = ["TOOLCHAIN_VERSION"] + includes = ["TOOLCHAIN_VERSION", "COVERAGE"] test_cmd = get_subprocess_exec( include_expansions_in_env=includes, args=[".evergreen/just.sh", "test-numpy"] ) diff --git a/.evergreen/scripts/run_tests.py b/.evergreen/scripts/run_tests.py index 9c8101c5b1..fcf5fe76cd 100644 --- a/.evergreen/scripts/run_tests.py +++ b/.evergreen/scripts/run_tests.py @@ -4,7 +4,9 @@ import logging import os import platform +import shlex import shutil +import subprocess import sys from datetime import datetime from pathlib import Path @@ -202,6 +204,16 @@ def run() -> None: if os.environ.get("DEBUG_LOG"): TEST_ARGS.extend(f"-o log_cli_level={logging.DEBUG}".split()) + if os.environ.get("COVERAGE"): + binary = sys.executable.replace(os.sep, "/") + cmd = f"{binary} -m coverage run -m pytest {' '.join(TEST_ARGS)} {' '.join(sys.argv[1:])}" + result = subprocess.run(shlex.split(cmd), check=False) # noqa: S603 + cmd = f"{binary} -m coverage report" + subprocess.run(shlex.split(cmd), check=False) # noqa: S603 + if result.returncode != 0: + print(result.stderr) + sys.exit(result.returncode) + # Run local tests. ret = pytest.main(TEST_ARGS + sys.argv[1:]) if ret != 0: diff --git a/.evergreen/scripts/setup_tests.py b/.evergreen/scripts/setup_tests.py index 939423ffcc..f3d86973b4 100644 --- a/.evergreen/scripts/setup_tests.py +++ b/.evergreen/scripts/setup_tests.py @@ -431,6 +431,9 @@ def handle_test_env() -> None: # We do not want the default client_context to be initialized. write_env("DISABLE_CONTEXT") + if test_name == "numpy": + UV_ARGS.append("--with numpy") + if test_name == "perf": data_dir = ROOT / "specifications/source/benchmarking/data" if not data_dir.exists(): @@ -458,7 +461,6 @@ def handle_test_env() -> None: # Keep in sync with combine-coverage.sh. # coverage >=5 is needed for relative_files=true. UV_ARGS.append("--group coverage") - TEST_ARGS = f"{TEST_ARGS} --cov" write_env("COVERAGE") if opts.green_framework: diff --git a/.evergreen/scripts/upload-codecov.sh b/.evergreen/scripts/upload-codecov.sh index 75bd9d9e21..5c1d84c55c 100755 --- a/.evergreen/scripts/upload-codecov.sh +++ b/.evergreen/scripts/upload-codecov.sh @@ -8,18 +8,20 @@ ROOT=$(dirname "$(dirname $HERE)") pushd $ROOT > /dev/null export FNAME=coverage.xml - -if [ -n "${is_patch:-}" ]; then - echo "This is a patch build, not running codecov" - exit 0 -fi +REQUESTER=${requester:-} if [ ! -f ".coverage" ]; then echo "There are no coverage results, not running codecov" exit 0 fi -echo "Uploading..." +if [[ "${REQUESTER}" == "github_pr" || "${REQUESTER}" == "commit" ]]; then + echo "Uploading codecov for $REQUESTER..." +else + echo "Error: requester must be 'github_pr' or 'commit', got '${REQUESTER}'" >&2 + exit 1 +fi + printf 'sha: %s\n' "$github_commit" printf 'flag: %s-%s\n' "$build_variant" "$task_name" printf 'file: %s\n' "$FNAME" @@ -40,18 +42,16 @@ codecov_args=( if [ -n "${github_pr_number:-}" ]; then printf 'branch: %s:%s\n' "$github_author" "$github_pr_head_branch" printf 'pr: %s\n' "$github_pr_number" - uv tool run --from codecov-cli codecovcli \ "${codecov_args[@]}" \ --pr "${github_pr_number}" \ --branch "${github_author}:${github_pr_head_branch}" else printf 'branch: %s\n' "$branch_name" - uv tool run --from codecov-cli codecovcli \ "${codecov_args[@]}" \ --branch "${branch_name}" fi -echo "Uploading...done." +echo "Uploading codecov for $REQUESTER... done." popd > /dev/null diff --git a/.evergreen/scripts/utils.py b/.evergreen/scripts/utils.py index 2bc9c720d2..02238645ce 100644 --- a/.evergreen/scripts/utils.py +++ b/.evergreen/scripts/utils.py @@ -44,6 +44,7 @@ class Distro: "mockupdb": "mockupdb", "ocsp": "ocsp", "perf": "perf", + "numpy": "", } # Tests that require a sub test suite. diff --git a/justfile b/justfile index 082b6ea170..82b1ac91dc 100644 --- a/justfile +++ b/justfile @@ -60,8 +60,9 @@ test *args="-v --durations=5 --maxfail=10": && resync uv run --extra test python -m pytest {{args}} [group('test')] -test-numpy: && resync - uv run --extra test --with numpy python -m pytest test/test_bson.py +test-numpy *args="": && resync + just setup-tests numpy {{args}} + just run-tests test/test_bson.py [group('test')] run-tests *args: && resync diff --git a/pyproject.toml b/pyproject.toml index acc9fa5b0d..9b3287834a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,6 @@ dev = [] pip = ["pip>=20.2"] gevent = ["gevent>=21.12"] coverage = [ - "pytest-cov>=4.0.0", "coverage[toml]>=5,<=7.10.7" ] mockupdb = [ diff --git a/test/asynchronous/test_ssl.py b/test/asynchronous/test_ssl.py index 0ce3e8bbac..7fe57e8503 100644 --- a/test/asynchronous/test_ssl.py +++ b/test/asynchronous/test_ssl.py @@ -48,19 +48,11 @@ _HAVE_PYOPENSSL = False try: - # All of these must be available to use PyOpenSSL - import OpenSSL - import requests - import service_identity - - # Ensure service_identity>=18.1 is installed - from service_identity.pyopenssl import verify_ip_address - - from pymongo.ocsp_support import _load_trusted_ca_certs + from pymongo import pyopenssl_context _HAVE_PYOPENSSL = True except ImportError: - _load_trusted_ca_certs = None # type: ignore + pass if HAVE_SSL: @@ -136,11 +128,6 @@ def test_config_ssl(self): def test_use_pyopenssl_when_available(self): self.assertTrue(HAVE_PYSSL) - @unittest.skipUnless(_HAVE_PYOPENSSL, "Cannot test without PyOpenSSL") - def test_load_trusted_ca_certs(self): - trusted_ca_certs = _load_trusted_ca_certs(CA_BUNDLE_PEM) - self.assertEqual(2, len(trusted_ca_certs)) - class TestSSL(AsyncIntegrationTest): saved_port: int diff --git a/test/test_ssl.py b/test/test_ssl.py index b1e9a65eb5..77bb086ecb 100644 --- a/test/test_ssl.py +++ b/test/test_ssl.py @@ -48,19 +48,11 @@ _HAVE_PYOPENSSL = False try: - # All of these must be available to use PyOpenSSL - import OpenSSL - import requests - import service_identity - - # Ensure service_identity>=18.1 is installed - from service_identity.pyopenssl import verify_ip_address - - from pymongo.ocsp_support import _load_trusted_ca_certs + from pymongo import pyopenssl_context _HAVE_PYOPENSSL = True except ImportError: - _load_trusted_ca_certs = None # type: ignore + pass if HAVE_SSL: @@ -136,11 +128,6 @@ def test_config_ssl(self): def test_use_pyopenssl_when_available(self): self.assertTrue(HAVE_PYSSL) - @unittest.skipUnless(_HAVE_PYOPENSSL, "Cannot test without PyOpenSSL") - def test_load_trusted_ca_certs(self): - trusted_ca_certs = _load_trusted_ca_certs(CA_BUNDLE_PEM) - self.assertEqual(2, len(trusted_ca_certs)) - class TestSSL(IntegrationTest): saved_port: int diff --git a/uv.lock b/uv.lock index 7d0ff9fb0f..78d0cc213f 100644 --- a/uv.lock +++ b/uv.lock @@ -1562,7 +1562,6 @@ zstd = [ [package.dev-dependencies] coverage = [ { name = "coverage", extra = ["toml"] }, - { name = "pytest-cov" }, ] gevent = [ { name = "gevent" }, @@ -1612,10 +1611,7 @@ requires-dist = [ provides-extras = ["aws", "docs", "encryption", "gssapi", "ocsp", "snappy", "test", "zstd"] [package.metadata.requires-dev] -coverage = [ - { name = "coverage", extras = ["toml"], specifier = ">=5,<=7.10.7" }, - { name = "pytest-cov", specifier = ">=4.0.0" }, -] +coverage = [{ name = "coverage", extras = ["toml"], specifier = ">=5,<=7.10.7" }] dev = [] gevent = [{ name = "gevent", specifier = ">=21.12" }] mockupdb = [{ name = "mockupdb", git = "https://github.com/mongodb-labs/mongo-mockup-db?rev=master" }] @@ -1763,21 +1759,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl", hash = "sha256:611e26147c7f77640e6d0a92a38ed17c3e9848063698d5c93d5aa7aa11cebff5", size = 15075, upload-time = "2025-11-10T16:07:45.537Z" }, ] -[[package]] -name = "pytest-cov" -version = "7.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "coverage", extra = ["toml"] }, - { name = "pluggy" }, - { name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "pytest", version = "9.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5e/f7/c933acc76f5208b3b00089573cf6a2bc26dc80a8aece8f52bb7d6b1855ca/pytest_cov-7.0.0.tar.gz", hash = "sha256:33c97eda2e049a0c5298e91f519302a1334c26ac65c1a483d6206fd458361af1", size = 54328, upload-time = "2025-09-09T10:57:02.113Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ee/49/1377b49de7d0c1ce41292161ea0f721913fa8722c19fb9c1e3aa0367eecb/pytest_cov-7.0.0-py3-none-any.whl", hash = "sha256:3b8e9558b16cc1479da72058bdecf8073661c7f57f7d3c5f22a1c23507f2d861", size = 22424, upload-time = "2025-09-09T10:57:00.695Z" }, -] - [[package]] name = "python-dateutil" version = "2.9.0.post0"