(Working Yesterday!) SIte-Specific Issue

[barrettruth]

Attempting to scraper codeforces, e.g. https://codeforces.com/contest/1335/problem/f

Expected Result

As of yesterday, the scraper worked - perhaps due to site upgrades, the scraper is now 403 blocked.

Actual Result

403 - permuting over many possible scraper configs still didn't work :/

#!/usr/bin/env python3
import sys, time
import cloudscraper

URL = "https://codeforces.com/contest/1335/problem/f"

browsers = ["chrome", "firefox"]
interpreters = ["js2py", "nodejs", "native", None]
stealth_flags = [False, True]

for b in browsers:
    for interp in interpreters:
        for ste in stealth_flags:
            try:
                kw = {
                    "delay": 5,
                    "enable_stealth": ste,
                    "debug": True,
                    "browser": {"browser": b, "platform": "linux", "mobile": False}
                }
                if interp:
                    kw["interpreter"] = interp
                s = cloudscraper.create_scraper(**kw)
                r = s.get(URL, timeout=(10, 30))
                if r.status_code == 200 and "problem-statement" in r.text:
                    print(r.text)
                    sys.exit(0)
            except Exception as e:
                time.sleep(0.5)

sys.exit(1)

System Information

$ python -m cloudscraper.help

{
    "platform": {
        "system": "Linux",
        "release": "6.16.8-arch1-1"
    },
    "interpreter": {
        "name": "CPython",
        "version": "3.11.12"
    },
    "cloudscraper": "1.2.71",
    "requests": "2.32.5",
    "urllib3": "2.5.0",
    "OpenSSL": {
        "version": "OpenSSL 3.0.16 11 Feb 2025",
        "ciphers": [
            "ADH-AES128-GCM-SHA256",
            "ADH-AES128-SHA",
            "ADH-AES128-SHA256",
            "ADH-AES256-GCM-SHA384",
            "ADH-AES256-SHA",
            "ADH-AES256-SHA256",
            "ADH-CAMELLIA128-SHA",
            "ADH-CAMELLIA128-SHA256",
            "ADH-CAMELLIA256-SHA",
            "ADH-CAMELLIA256-SHA256",
            "AECDH-AES128-SHA",
            "AECDH-AES256-SHA",
            "AES128-CCM",
            "AES128-CCM8",
            "AES128-GCM-SHA256",
            "AES128-SHA",
            "AES128-SHA256",
            "AES256-CCM",
            "AES256-CCM8",
            "AES256-GCM-SHA384",
            "AES256-SHA",
            "AES256-SHA256",
            "ARIA128-GCM-SHA256",
            "ARIA256-GCM-SHA384",
            "CAMELLIA128-SHA",
            "CAMELLIA128-SHA256",
            "CAMELLIA256-SHA",
            "CAMELLIA256-SHA256",
            "DHE-DSS-AES128-GCM-SHA256",
            "DHE-DSS-AES128-SHA",
            "DHE-DSS-AES128-SHA256",
            "DHE-DSS-AES256-GCM-SHA384",
            "DHE-DSS-AES256-SHA",
            "DHE-DSS-AES256-SHA256",
            "DHE-DSS-ARIA128-GCM-SHA256",
            "DHE-DSS-ARIA256-GCM-SHA384",
            "DHE-DSS-CAMELLIA128-SHA",
            "DHE-DSS-CAMELLIA128-SHA256",
            "DHE-DSS-CAMELLIA256-SHA",
            "DHE-DSS-CAMELLIA256-SHA256",
            "DHE-PSK-AES128-CBC-SHA",
            "DHE-PSK-AES128-CBC-SHA256",
            "DHE-PSK-AES128-CCM",
            "DHE-PSK-AES128-CCM8",
            "DHE-PSK-AES128-GCM-SHA256",
            "DHE-PSK-AES256-CBC-SHA",
            "DHE-PSK-AES256-CBC-SHA384",
            "DHE-PSK-AES256-CCM",
            "DHE-PSK-AES256-CCM8",
            "DHE-PSK-AES256-GCM-SHA384",
            "DHE-PSK-ARIA128-GCM-SHA256",
            "DHE-PSK-ARIA256-GCM-SHA384",
            "DHE-PSK-CAMELLIA128-SHA256",
            "DHE-PSK-CAMELLIA256-SHA384",
            "DHE-PSK-CHACHA20-POLY1305",
            "DHE-RSA-AES128-CCM",
            "DHE-RSA-AES128-CCM8",
            "DHE-RSA-AES128-GCM-SHA256",
            "DHE-RSA-AES128-SHA",
            "DHE-RSA-AES128-SHA256",
            "DHE-RSA-AES256-CCM",
            "DHE-RSA-AES256-CCM8",
            "DHE-RSA-AES256-GCM-SHA384",
            "DHE-RSA-AES256-SHA",
            "DHE-RSA-AES256-SHA256",
            "DHE-RSA-ARIA128-GCM-SHA256",
            "DHE-RSA-ARIA256-GCM-SHA384",
            "DHE-RSA-CAMELLIA128-SHA",
            "DHE-RSA-CAMELLIA128-SHA256",
            "DHE-RSA-CAMELLIA256-SHA",
            "DHE-RSA-CAMELLIA256-SHA256",
            "DHE-RSA-CHACHA20-POLY1305",
            "ECDHE-ARIA128-GCM-SHA256",
            "ECDHE-ARIA256-GCM-SHA384",
            "ECDHE-ECDSA-AES128-CCM",
            "ECDHE-ECDSA-AES128-CCM8",
            "ECDHE-ECDSA-AES128-GCM-SHA256",
            "ECDHE-ECDSA-AES128-SHA",
            "ECDHE-ECDSA-AES128-SHA256",
            "ECDHE-ECDSA-AES256-CCM",
            "ECDHE-ECDSA-AES256-CCM8",
            "ECDHE-ECDSA-AES256-GCM-SHA384",
            "ECDHE-ECDSA-AES256-SHA",
            "ECDHE-ECDSA-AES256-SHA384",
            "ECDHE-ECDSA-ARIA128-GCM-SHA256",
            "ECDHE-ECDSA-ARIA256-GCM-SHA384",
            "ECDHE-ECDSA-CAMELLIA128-SHA256",
            "ECDHE-ECDSA-CAMELLIA256-SHA384",
            "ECDHE-ECDSA-CHACHA20-POLY1305",
            "ECDHE-PSK-AES128-CBC-SHA",
            "ECDHE-PSK-AES128-CBC-SHA256",
            "ECDHE-PSK-AES256-CBC-SHA",
            "ECDHE-PSK-AES256-CBC-SHA384",
            "ECDHE-PSK-CAMELLIA128-SHA256",
            "ECDHE-PSK-CAMELLIA256-SHA384",
            "ECDHE-PSK-CHACHA20-POLY1305",
            "ECDHE-RSA-AES128-GCM-SHA256",
            "ECDHE-RSA-AES128-SHA",
            "ECDHE-RSA-AES128-SHA256",
            "ECDHE-RSA-AES256-GCM-SHA384",
            "ECDHE-RSA-AES256-SHA",
            "ECDHE-RSA-AES256-SHA384",
            "ECDHE-RSA-CAMELLIA128-SHA256",
            "ECDHE-RSA-CAMELLIA256-SHA384",
            "ECDHE-RSA-CHACHA20-POLY1305",
            "PSK-AES128-CBC-SHA",
            "PSK-AES128-CBC-SHA256",
            "PSK-AES128-CCM",
            "PSK-AES128-CCM8",
            "PSK-AES128-GCM-SHA256",
            "PSK-AES256-CBC-SHA",
            "PSK-AES256-CBC-SHA384",
            "PSK-AES256-CCM",
            "PSK-AES256-CCM8",
            "PSK-AES256-GCM-SHA384",
            "PSK-ARIA128-GCM-SHA256",
            "PSK-ARIA256-GCM-SHA384",
            "PSK-CAMELLIA128-SHA256",
            "PSK-CAMELLIA256-SHA384",
            "PSK-CHACHA20-POLY1305",
            "RSA-PSK-AES128-CBC-SHA",
            "RSA-PSK-AES128-CBC-SHA256",
            "RSA-PSK-AES128-GCM-SHA256",
            "RSA-PSK-AES256-CBC-SHA",
            "RSA-PSK-AES256-CBC-SHA384",
            "RSA-PSK-AES256-GCM-SHA384",
            "RSA-PSK-ARIA128-GCM-SHA256",
            "RSA-PSK-ARIA256-GCM-SHA384",
            "RSA-PSK-CAMELLIA128-SHA256",
            "RSA-PSK-CAMELLIA256-SHA384",
            "RSA-PSK-CHACHA20-POLY1305",
            "SRP-AES-128-CBC-SHA",
            "SRP-AES-256-CBC-SHA",
            "SRP-DSS-AES-128-CBC-SHA",
            "SRP-DSS-AES-256-CBC-SHA",
            "SRP-RSA-AES-128-CBC-SHA",
            "SRP-RSA-AES-256-CBC-SHA",
            "TLS_AES_128_GCM_SHA256",
            "TLS_AES_256_GCM_SHA384",
            "TLS_CHACHA20_POLY1305_SHA256"
        ]
    }
}

Debug Log

This command is only available on cloudscraper v1.2.33 and greater. Otherwise,
please provide some basic information about your system (Python version,
operating system).

Arch Linux

$ uname -a
Linux xps 6.16.8-arch1-1 #1 SMP PREEMPT_DYNAMIC ... +0000 x86_64 GNU/Linux

[barrettruth]

Thanks so much for the tool - even if this is impossible to get around it was very useful for my plugin cp.nvim while it lasted 😢