From 12d3ef481591b52d5f5e416c8c76d9fdbe80b096 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 11 Oct 2026 22:49:05 +0000 Subject: [PATCH] Fix stale request cache, auth resets, and download() error/session handling - cache_get(): replace the never-expiring lru_cache with a bounded cache whose entries expire after yf.config.network.cache_ttl seconds (default 60; 0 disables, None never expires). Error responses are not cached. Adds YfData.cache_clear(). - _make_request(): a 404 (delisted/mistyped ticker) no longer switches cookie strategy and wipes the crumb shared by all threads. - _get_cookie_basic(): mark the cookie as fetched, so fc.yahoo.com isn't re-requested on every call when the persistent cookie cache is unusable. - _set_session(): reset cookie/crumb when the session object changes. - download(): only replace the shared session when the caller passes one; request exceptions per call via PriceHistory._history(raise_errors=True) instead of toggling a config key nobody read (network.hide_exceptions), which would also have raced between threads. - Tickers.download(): drop dead assignment to an unused _history attribute. - Add offline regression tests (tests/test_reliability_fixes.py) and document cache_ttl. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VxuCGhah3SL3ZWQrwXUQDR --- doc/source/advanced/config.rst | 15 +- tests/test_reliability_fixes.py | 291 ++++++++++++++++++++++++++++++++ yfinance/config.py | 1 + yfinance/data.py | 66 +++++++- yfinance/multi.py | 34 ++-- yfinance/scrapers/history.py | 17 +- yfinance/tickers.py | 3 - 7 files changed, 399 insertions(+), 28 deletions(-) create mode 100644 tests/test_reliability_fixes.py diff --git a/doc/source/advanced/config.rst b/doc/source/advanced/config.rst index 992018954..1d3bc2e5d 100644 --- a/doc/source/advanced/config.rst +++ b/doc/source/advanced/config.rst @@ -11,7 +11,8 @@ Config { "network": { "proxy": null, - "retries": 0 + "retries": 0, + "cache_ttl": 60 }, "debug": { "hide_exceptions": true, @@ -21,7 +22,8 @@ Config >>> yf.config.network { "proxy": null, - "retries": 0 + "retries": 0, + "cache_ttl": 60 } @@ -40,6 +42,15 @@ Network yf.config.network.retries = 2 +* **cache_ttl** - Seconds that repeated identical requests reuse a cached + response (default 60). ``0`` disables the cache, ``None`` never expires + entries. Error responses are never cached. + + .. code-block:: python + + yf.config.network.cache_ttl = 10 # live service: keep data fresh + yf.config.network.cache_ttl = None # backtest: fetch once per process + Debug ----- diff --git a/tests/test_reliability_fixes.py b/tests/test_reliability_fixes.py new file mode 100644 index 000000000..9c4406bcc --- /dev/null +++ b/tests/test_reliability_fixes.py @@ -0,0 +1,291 @@ +"""Offline regression tests for request-cache, cookie/crumb, session and +download() error-handling fixes. No network access needed.""" +import unittest +import warnings +from unittest import mock + +import pandas as pd + +from tests.context import yfinance as yf +from yfinance import multi +from yfinance._http import new_session +from yfinance.config import YfConfig +from yfinance.data import SingletonMeta, YfData +from yfinance.scrapers.history import PriceHistory + +CHART_URL = "https://query2.finance.yahoo.com/v8/finance/chart/AAPL" +MARKET_URL = "https://query1.finance.yahoo.com/v6/finance/quote/marketSummary" + + +def _mock_response(status_code=200, text="{}", url=CHART_URL): + resp = mock.MagicMock() + resp.status_code = status_code + resp.text = text + resp.content = text.encode() or b"" + resp.url = url + resp.json.return_value = {} + return resp + + +class _FreshYfData(unittest.TestCase): + """YfData is a singleton: give each test its own instance and a fake + network, so no state leaks between tests.""" + + def setUp(self): + SingletonMeta._instances.pop(YfData, None) + self._ttl = YfConfig.network.cache_ttl + self._no_persistent_cookie = mock.patch.object( + YfData, '_load_cookie_curlCffi', return_value=False) + self._no_persistent_cookie.start() + self._no_cookie_save = mock.patch.object( + YfData, '_save_cookie_curlCffi', return_value=True) + self._no_cookie_save.start() + self.data = YfData() + self.requests = [] + self.status_for = {} # url substring -> status code + + def fake_get(url=None, **kwargs): + self.requests.append(url) + if 'getcrumb' in url: + return _mock_response(200, 'CRUMB', url=url) + for part, code in self.status_for.items(): + if part in url: + return _mock_response(code, url=url) + return _mock_response(200, url=url) + + self.data._session.get = mock.MagicMock(side_effect=fake_get) + self.data._session.post = mock.MagicMock(side_effect=fake_get) + + def tearDown(self): + YfConfig.network.cache_ttl = self._ttl + self._no_persistent_cookie.stop() + self._no_cookie_save.stop() + SingletonMeta._instances.pop(YfData, None) + + def target_requests(self, part): + return [u for u in self.requests if part in u] + + +class TestCacheExpiry(_FreshYfData): + """Bug 1: cache_get() used to cache responses forever.""" + + def test_repeat_within_ttl_uses_cache(self): + YfConfig.network.cache_ttl = 60 + self.data.cache_get(MARKET_URL, params={'market': 'US'}) + self.data.cache_get(MARKET_URL, params={'market': 'US'}) + self.assertEqual(len(self.target_requests('marketSummary')), 1) + + def test_expired_entry_is_refetched(self): + YfConfig.network.cache_ttl = 60 + with mock.patch('yfinance.data._time.monotonic', return_value=1000.0): + self.data.cache_get(MARKET_URL, params={'market': 'US'}) + with mock.patch('yfinance.data._time.monotonic', return_value=1061.0): + self.data.cache_get(MARKET_URL, params={'market': 'US'}) + self.assertEqual(len(self.target_requests('marketSummary')), 2) + + def test_error_responses_are_not_cached(self): + self.status_for['marketSummary'] = 500 + self.data.cache_get(MARKET_URL) + self.data.cache_get(MARKET_URL) + # Each call reaches the network (500 also triggers one auth retry). + self.assertEqual(len(self.target_requests('marketSummary')), 4) + + def test_ttl_zero_disables_cache(self): + YfConfig.network.cache_ttl = 0 + self.data.cache_get(MARKET_URL) + self.data.cache_get(MARKET_URL) + self.assertEqual(len(self.target_requests('marketSummary')), 2) + + def test_ttl_none_never_expires(self): + YfConfig.network.cache_ttl = None + with mock.patch('yfinance.data._time.monotonic', return_value=0.0): + self.data.cache_get(MARKET_URL) + with mock.patch('yfinance.data._time.monotonic', return_value=1e9): + self.data.cache_get(MARKET_URL) + self.assertEqual(len(self.target_requests('marketSummary')), 1) + + def test_cache_clear(self): + self.data.cache_get(MARKET_URL) + self.data.cache_clear() + self.data.cache_get(MARKET_URL) + self.assertEqual(len(self.target_requests('marketSummary')), 2) + + def test_cache_size_is_bounded(self): + YfConfig.network.cache_ttl = None + for i in range(100): + self.data.cache_get(MARKET_URL, params={'i': i}) + self.assertLessEqual(len(self.data._response_cache), 64) + + def test_default_ttl_is_set(self): + self.assertEqual(YfConfig.network.cache_ttl, 60) + + +class TestNotFoundKeepsAuth(_FreshYfData): + """Bug 2: a 404 (bad/delisted ticker) used to flip cookie strategy and + wipe the crumb shared by every thread.""" + + def test_404_does_not_toggle_strategy_or_drop_crumb(self): + self.data.get(CHART_URL) # warm up cookie + crumb + strategy, crumb = self.data._cookie_strategy, self.data._crumb + self.requests.clear() + + self.status_for['BADTICKER'] = 404 + resp = self.data.get("https://query2.finance.yahoo.com/v8/finance/chart/BADTICKER") + + self.assertEqual(resp.status_code, 404) + self.assertEqual(self.data._cookie_strategy, strategy) + self.assertEqual(self.data._crumb, crumb) + self.assertEqual(self.requests, ["https://query2.finance.yahoo.com/v8/finance/chart/BADTICKER"]) + + def test_401_still_retries_with_fresh_auth(self): + self.data.get(CHART_URL) + self.requests.clear() + + self.status_for['quoteSummary'] = 401 + self.data.get("https://query2.finance.yahoo.com/v10/finance/quoteSummary/AAPL") + + # Auth was reset and re-fetched, then the request retried once. + self.assertTrue(self.target_requests('getcrumb')) + self.assertEqual(len(self.target_requests('quoteSummary')), 2) + + +class TestCookieReuse(_FreshYfData): + """Bug 6 (found while testing): with the default 'basic' strategy the + fetched cookie was never marked, so every request re-hit fc.yahoo.com + whenever the persistent cookie cache couldn't be used.""" + + def test_cookie_fetched_once_across_requests(self): + for _ in range(3): + self.data.get(CHART_URL) + self.assertEqual(len(self.target_requests('fc.yahoo.com')), 1) + self.assertEqual(len(self.target_requests('getcrumb')), 1) + + +def _fake_download_one(ctx, ticker, *args, **kwargs): + with ctx.lock: + ctx.dfs[ticker.upper()] = pd.DataFrame( + {'Close': [1.0]}, index=pd.DatetimeIndex(['2024-01-02'])) + + +class TestSessionSwap(_FreshYfData): + """Bug 4: download() replaced the shared session on every call and the + old crumb was kept with the new (empty) cookie jar.""" + + def test_new_session_resets_cookie_and_crumb(self): + self.data._cookie, self.data._crumb = True, 'OLD' + new = new_session() + YfData(session=new) + self.assertIs(self.data._session, new) + self.assertIsNone(self.data._crumb) + self.assertIsNone(self.data._cookie) + + def test_same_session_keeps_crumb(self): + self.data._crumb = 'KEEP' + YfData(session=self.data._session) + self.assertEqual(self.data._crumb, 'KEEP') + + def test_download_without_session_keeps_shared_session(self): + session_before = self.data._session + with mock.patch.object(multi, '_download_one', side_effect=_fake_download_one): + yf.download('AAPL', threads=False, progress=False) + self.assertIs(self.data._session, session_before) + + def test_download_with_session_uses_it(self): + mine = new_session() + with mock.patch.object(multi, '_download_one', side_effect=_fake_download_one): + yf.download('AAPL', threads=False, progress=False, session=mine) + self.assertIs(self.data._session, mine) + + +class _FakePriceHistory: + def __init__(self, error=None, last_error=None): + self.error = error + self._last_error = last_error + self.calls = [] + + def _history(self, **kwargs): + self.calls.append((kwargs, YfConfig.debug.hide_exceptions)) + if self.error: + raise self.error + return pd.DataFrame({'Close': [1.0]}, + index=pd.DatetimeIndex(['2024-01-02'], tz='America/New_York')) + + +class TestDownloadErrors(unittest.TestCase): + """Bug 3: download() toggled a config key nobody reads + (network.hide_exceptions), so errors were never raised into its + collector; fixing the key name alone would race between threads.""" + + def setUp(self): + self._hide = YfConfig.debug.hide_exceptions + + def tearDown(self): + YfConfig.debug.hide_exceptions = self._hide + + def _run(self, fake): + ctx = multi._DownloadCtx() + with mock.patch('yfinance.base.TickerBase._lazy_load_price_history', return_value=fake): + multi._download_one(ctx, 'AAPL', period='1mo') + return ctx + + def test_requests_raise_without_touching_global_config(self): + fake = _FakePriceHistory() + self._run(fake) + kwargs, hide_during_call = fake.calls[0] + self.assertTrue(kwargs['raise_errors']) + self.assertTrue(hide_during_call) # global left alone during the call + self.assertTrue(YfConfig.debug.hide_exceptions) + self.assertNotIn('hide_exceptions', YfConfig.network.data) + + def test_exception_is_collected_with_traceback(self): + ctx = self._run(_FakePriceHistory(error=ValueError('boom'))) + self.assertIn('boom', ctx.errors['AAPL']) + self.assertIn('ValueError', ctx.tracebacks['AAPL']) + self.assertTrue(ctx.dfs['AAPL'].empty) + + def test_clean_message_preferred_over_repr(self): + fake = _FakePriceHistory(error=yf.exceptions.YFTzMissingError('$XYZ'), + last_error='possibly delisted; no timezone found') + ctx = self._run(fake) + self.assertEqual(ctx.errors['AAPL'], 'possibly delisted; no timezone found') + + def test_download_logs_failures_once(self): + fake = _FakePriceHistory(error=ValueError('boom')) + with mock.patch('yfinance.base.TickerBase._lazy_load_price_history', return_value=fake), \ + self.assertLogs('yfinance', level='ERROR') as logs: + df = yf.download('AAPL', threads=False, progress=False) + self.assertTrue(df.empty or df.isna().all().all()) + self.assertTrue(any('boom' in m for m in logs.output)) + + def test_public_history_still_warns_on_raise_errors(self): + ph = PriceHistory.__new__(PriceHistory) + with mock.patch.object(PriceHistory, '_history', return_value='ok') as impl, \ + warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always') + self.assertEqual(ph.history(raise_errors=True), 'ok') + self.assertTrue(any(issubclass(w.category, DeprecationWarning) for w in caught)) + self.assertTrue(impl.call_args.kwargs['raise_errors']) + + +class TestTickersDownload(unittest.TestCase): + """Bug 5: Tickers.download() stored results on an attribute nothing + reads (and its {} fallback would crash).""" + + def _fake_download(self, symbols, **kwargs): + idx = pd.DatetimeIndex(['2024-01-02']) + cols = pd.MultiIndex.from_product([symbols, ['Close', 'Open']], names=['Ticker', 'Price']) + return pd.DataFrame([[1.0, 2.0] * len(symbols)], index=idx, columns=cols) + + def test_group_by_column_and_ticker(self): + with mock.patch.object(multi, 'download', side_effect=self._fake_download): + t = yf.Tickers('AAPL MSFT') + by_col = t.download(group_by='column', progress=False) + by_tkr = t.download(group_by='ticker', progress=False) + self.assertEqual(list(by_col.columns.get_level_values(0).unique()), ['Close', 'Open']) + self.assertEqual(sorted(by_tkr.columns.get_level_values(0).unique()), ['AAPL', 'MSFT']) + for tkr in t.tickers.values(): + self.assertFalse(hasattr(tkr, '_history')) + + +if __name__ == '__main__': + unittest.main() diff --git a/yfinance/config.py b/yfinance/config.py index 8f5d30d98..aa7fef896 100644 --- a/yfinance/config.py +++ b/yfinance/config.py @@ -30,6 +30,7 @@ def _load_option(self): n = self.__getattr__('network') n.proxy = None n.retries = 0 + n.cache_ttl = 60 # seconds to reuse cached GET responses; 0 = no caching, None = never expire d = self.__getattr__('debug') d.hide_exceptions = True d.logging = False diff --git a/yfinance/data.py b/yfinance/data.py index 9d8d56957..f5541d1e4 100644 --- a/yfinance/data.py +++ b/yfinance/data.py @@ -1,5 +1,5 @@ import functools -from functools import lru_cache +from collections import OrderedDict import socket import time as _time @@ -36,6 +36,13 @@ def _normalize_proxy(proxy): return proxy +def _freeze_params(params): + """Hashable form of a request params dict, for use as a cache key.""" + if params is None: + return None + return frozendict({k: tuple(v) if isinstance(v, list) else v for k, v in params.items()}) + + def lru_cache_freezeargs(func): """ Decorator transforms mutable dictionary and list arguments into immutable types @@ -105,6 +112,10 @@ def __init__(self, session=None): # ticker; reverted if the chunked fallback also fails. self.fundamentals_use_chunked: bool = False + # cache_get() responses: key -> (monotonic time stored, response) + self._response_cache = OrderedDict() + self._response_cache_lock = threading.Lock() + self._session = None self._set_session(session or new_session()) @@ -146,6 +157,11 @@ def _set_session(self, session): raise YFDataException(f"Unsupported session type {type(session)}; expected curl_cffi or requests Session. Solution: stop setting session, let yfinance handle.") with self._cookie_lock: + if self._session is not None and session is not self._session: + # The cookie and crumb belong to the old session's cookie jar; + # sending the old crumb with the new session's jar fails auth. + self._cookie = None + self._crumb = None self._session = session if YfConfig.network.proxy is not None: self._session.proxies = _normalize_proxy(YfConfig.network.proxy) @@ -252,6 +268,9 @@ def _get_cookie_basic(self, timeout=30): f"Cookie fetch from fc.yahoo.com failed ({type(e).__name__}), continuing without it") return False raise + # Mark the cookie as fetched (as _get_cookie_csrf does), else every + # request re-fetches fc.yahoo.com when the persistent cache is unusable. + self._cookie = True self._save_cookie_curlCffi() return True @@ -480,7 +499,10 @@ def _make_request(self, url, request_method, body=None, params=None, timeout=30, else: raise utils.get_yf_logger().debug(f'response code={response.status_code}') - if response.status_code >= 400: + # 404 means the resource doesn't exist (e.g. delisted or mistyped + # ticker), not an auth failure: switching cookie strategy can't help, + # and would wipe the cookie/crumb shared by every thread. + if response.status_code >= 400 and response.status_code != 404: # Retry with other cookie strategy try: if strategy == 'basic': @@ -507,10 +529,44 @@ def _make_request(self, url, request_method, body=None, params=None, timeout=30, return response - @lru_cache_freezeargs - @lru_cache(maxsize=cache_maxsize) def cache_get(self, url, params=None, timeout=30): - return self.get(url, params, timeout) + """GET through a small in-memory cache. + + Entries expire after ``yf.config.network.cache_ttl`` seconds so + long-running processes don't serve stale data forever (0 disables + caching, None never expires). Error responses are not cached. + """ + ttl = YfConfig.network.cache_ttl + if ttl == 0: + return self.get(url, params, timeout) + try: + key = (url, _freeze_params(params), timeout) + hash(key) + except TypeError: + return self.get(url, params, timeout) + + with self._response_cache_lock: + hit = self._response_cache.get(key) + if hit is not None: + stored_at, response = hit + if ttl is None or _time.monotonic() - stored_at < ttl: + self._response_cache.move_to_end(key) + return response + del self._response_cache[key] + + response = self.get(url, params, timeout) + if response.status_code < 400: + with self._response_cache_lock: + self._response_cache[key] = (_time.monotonic(), response) + self._response_cache.move_to_end(key) + while len(self._response_cache) > cache_maxsize: + self._response_cache.popitem(last=False) + return response + + def cache_clear(self): + """Drop all responses cached by cache_get().""" + with self._response_cache_lock: + self._response_cache.clear() def get_raw_json(self, url, params=None, timeout=30): utils.get_yf_logger().debug(f'get_raw_json(): {url}') diff --git a/yfinance/multi.py b/yfinance/multi.py index 2a5c368b9..d981df1ee 100644 --- a/yfinance/multi.py +++ b/yfinance/multi.py @@ -30,11 +30,9 @@ import multitasking as _multitasking import pandas as _pd import numpy as _np -from ._http import new_session from . import Ticker, utils from .data import YfData -from .config import YfConfig from .const import period_default @@ -128,9 +126,8 @@ def _download_impl(ctx, tickers, start=None, end=None, actions=False, threads=Tr prepost=False, rounding=False, timeout=10, session=None, multi_level_index=True): logger = utils.get_yf_logger() - session = session or new_session() - - YfData(session=session) + if session is not None: + YfData(session=session) if logger.isEnabledFor(logging.DEBUG): if threads: @@ -277,30 +274,35 @@ def _download_one(ctx, ticker, start=None, end=None, data = None sym = ticker.upper() - backup = YfConfig.network.hide_exceptions - YfConfig.network.hide_exceptions = False + ph = None try: tkr = Ticker(ticker) - data = tkr.history( + ph = tkr._lazy_load_price_history() + # Raise per call so failures are collected into ctx.errors (and logged + # once, grouped, by download()) instead of being logged per ticker. + # Never toggle the global hide_exceptions config here: other threads + # read it concurrently. + data = ph._history( period=period, interval=interval, start=start, end=end, prepost=prepost, actions=actions, auto_adjust=auto_adjust, back_adjust=back_adjust, repair=repair, - rounding=rounding, keepna=keepna, timeout=timeout + rounding=rounding, keepna=keepna, timeout=timeout, + raise_errors=True ) with ctx.lock: ctx.dfs[sym] = data - # PriceHistory records soft errors (e.g. delisted, missing tz) - # without raising; surface them so download() can log them. - ph = tkr._price_history - if ph is not None and ph._last_error is not None: + if ph._last_error is not None: ctx.errors[sym] = ph._last_error except Exception as e: with ctx.lock: ctx.dfs[sym] = utils.empty_df() - ctx.errors[sym] = repr(e) + # Prefer the clean message PriceHistory recorded (e.g. "possibly + # delisted; no timezone found") over the exception repr. + if ph is not None and ph._last_error is not None: + ctx.errors[sym] = ph._last_error + else: + ctx.errors[sym] = repr(e) ctx.tracebacks[sym] = traceback.format_exc() - YfConfig.network.hide_exceptions = backup - return data diff --git a/yfinance/scrapers/history.py b/yfinance/scrapers/history.py index a7f0f98c1..2aee5b764 100644 --- a/yfinance/scrapers/history.py +++ b/yfinance/scrapers/history.py @@ -154,11 +154,24 @@ def history(self, period=period_default, interval="1d", raise_errors : bool If True, then raise errors as Exceptions instead of logging. """ - logger = utils.get_yf_logger() - if raise_errors: warnings.warn("'raise_errors' deprecated, do: yf.config.debug.hide_exceptions = False", DeprecationWarning, stacklevel=5) + return self._history(period=period, interval=interval, start=start, end=end, + prepost=prepost, actions=actions, auto_adjust=auto_adjust, + back_adjust=back_adjust, repair=repair, keepna=keepna, + rounding=rounding, timeout=timeout, raise_errors=raise_errors) + + def _history(self, period=period_default, interval="1d", + start=None, end=None, prepost=False, actions=True, + auto_adjust=True, back_adjust=False, repair=False, keepna=False, + rounding=False, timeout=10, + raise_errors=False) -> pd.DataFrame: + # history() without the raise_errors deprecation warning, so download() + # can request exceptions per call instead of flipping the global + # hide_exceptions config (which races between threads). + logger = utils.get_yf_logger() + interval_user = interval if period == period_default: period_user = None diff --git a/yfinance/tickers.py b/yfinance/tickers.py index 6f5a54e92..ba4d82180 100644 --- a/yfinance/tickers.py +++ b/yfinance/tickers.py @@ -80,9 +80,6 @@ def download(self, period='1mo if start & end None', interval="1d", timeout=timeout, **kwargs) - for symbol in self.symbols: - self.tickers.get(symbol, {})._history = data[symbol] - if group_by == 'column': data.columns = data.columns.swaplevel(0, 1) data.sort_index(level=0, axis=1, inplace=True)