Fix price-repair unit-switch sometimes applying twice

Author: ValueRaider

doc/source/advanced/price_repair.rst

@@ -6,6 +6,8 @@ The new argument ``repair=True`` in ``history()`` and ``download()`` will attemp
 
 The returned table will have a new column ``Repaired?`` that specifies if row was repaired.
 
+This can also repair currency mixups, so check ``Ticker.history_metadata['currency']`` after for true currency. Not ``Ticker.info``.
+
 Price repair
 ============
 

yfinance/scrapers/history.py

@@ -1088,6 +1088,7 @@ def _standardise_currency(self, df, currency):
             for c in _PRICE_COLNAMES_:
                 df[c] *= m
         self._history_metadata["currency"] = currency2
+        self._history_metadata["currencyRepaired"] = True
 
         f_div = df['Dividends']!=0.0
         if f_div.any():
@@ -1323,7 +1324,7 @@ def _fix_unit_switch(self, df, interval, tz_exchange):
             n = 1000
         else:
             n = 100
-        return self._fix_prices_sudden_change(df, interval, tz_exchange, n, correct_dividend=True)
+        return self._fix_prices_sudden_change(df, interval, tz_exchange, n, unit_switch=True, correct_dividend=True)
 
     @utils.log_indent_decorator
     def _fix_zeroes(self, df, interval, tz_exchange, prepost):
@@ -2656,7 +2657,7 @@ def _fix_bad_stock_splits(self, df, interval, tz_exchange):
         return df
 
     @utils.log_indent_decorator
-    def _fix_prices_sudden_change(self, df, interval, tz_exchange, change, correct_volume=False, correct_dividend=False):
+    def _fix_prices_sudden_change(self, df, interval, tz_exchange, change, unit_switch=False, correct_volume=False, correct_dividend=False):
         if df.empty:
             return df
 
@@ -2668,7 +2669,7 @@ def _fix_prices_sudden_change(self, df, interval, tz_exchange, change, correct_v
         interday = interval in ['1d', '1wk', '1mo', '3mo']
         multiday = interval in ['1wk', '1mo', '3mo']
 
-        if change in [100.0, 0.01]:
+        if unit_switch:
             fix_type = '100x error'
             log_extras['yf_cat'] = 'price-repair-100x'
             start_min = None
@@ -2834,8 +2835,7 @@ def _fix_prices_sudden_change(self, df, interval, tz_exchange, change, correct_v
         threshold = (split_max + 1.0 + largest_change_pct) * 0.5
         logger.debug(f"threshold={threshold:.3f}, threshold_rcp={1.0/threshold:.3f}", extra=log_extras)
 
-        if 'Repaired?' not in df2.columns:
-            df2['Repaired?'] = False
+        sudden_change_repaired = np.full(len(df2), False)
 
         if correct_columns_individually:
             _1d_change_x = np.full((n, 4), 1.0)
@@ -3124,6 +3124,8 @@ def map_signals_to_ranges(f, f_up, f_down):
 
             return ranges
 
+        any_m_lt_1 = False
+
         if idx_latest_active is not None:
             idx_rev_latest_active = df.shape[0] - 1 - idx_latest_active
             logger.debug(f'idx_latest_active={idx_latest_active}, idx_rev_latest_active={idx_rev_latest_active}', extra=log_extras)
@@ -3211,6 +3213,7 @@ def map_signals_to_ranges(f, f_up, f_down):
                         else:
                             m = split_rcp
                             m_rcp = split
+                        any_m_lt_1 = any_m_lt_1 or m < 0.99
                         if interday:
                             msg = f"Corrected {fix_type} on col={c} range=[{df2.index[r[1]-1].date()}:{df2.index[r[0]].date()}] m={m:.4f}"
                         else:
@@ -3245,7 +3248,7 @@ def map_signals_to_ranges(f, f_up, f_down):
                 if f_open_xor_closed_fixed.any():
                     df2.loc[f_open_xor_closed_fixed, "Volume"] = (df2.loc[f_open_xor_closed_fixed, "Volume"] * 0.5 * m_rcp).round().astype('int')
 
-            df2.loc[f_corrected, 'Repaired?'] = True
+            sudden_change_repaired[f_corrected] = True
 
         else:
             n_corrected = 0
@@ -3293,6 +3296,7 @@ def map_signals_to_ranges(f, f_up, f_down):
                 else:
                     m = split_rcp
                     m_rcp = split
+                any_m_lt_1 = any_m_lt_1 or m < 0.99
                 logger.debug(f"range={r} m={m}", extra=log_extras)
                 for c in ['Open', 'High', 'Low', 'Close', 'Adj Close']:
                     df2.iloc[r[0]:r[1], df2.columns.get_loc(c)] *= m
@@ -3301,7 +3305,7 @@ def map_signals_to_ranges(f, f_up, f_down):
                 if correct_volume:
                     col_loc = df2.columns.get_loc("Volume")
                     df2.iloc[r[0]:r[1], col_loc] = (df2.iloc[r[0]:r[1], col_loc] * m_rcp).round().astype('int')
-                df2.iloc[r[0]:r[1], df2.columns.get_loc('Repaired?')] = True
+                sudden_change_repaired[r[0]:r[1]] = True
                 if r[0] == r[1] - 1:
                     if interday:
                         msg = f"Corrected {fix_type} on interval {df2.index[r[0]].date()}"
@@ -3326,6 +3330,27 @@ def map_signals_to_ranges(f, f_up, f_down):
                 msg = f"Corrected: {n_corrected}x"
             logger.info(msg, extra=log_extras)
 
+        if unit_switch and any_m_lt_1:
+            # m < 1 means thats the switch was repaired in favour of the major currency
+            # e.g. USD beat cents
+            # But check if _standardise_currency() already did that.
+            if 'currencyRepaired' in self._history_metadata and self._history_metadata['currencyRepaired']:
+                # Yes it did, which means this repair did it again.
+                # Revert the second.
+                m = change
+                m_rcp = 1.0/change
+                for c in ['Open', 'High', 'Low', 'Close', 'Adj Close']:
+                    df2[c] *= m
+                if correct_dividend:
+                    df2['Dividends'] *= m
+                if correct_volume:
+                    df2['Volume'] = (df2['Volume'] * m_rcp).round().astype('int')
+                sudden_change_repaired = ~sudden_change_repaired
+
+        if 'Repaired?' not in df2.columns:
+            df2['Repaired?'] = False
+        df2['Repaired?'] = df2['Repaired?'].to_numpy() | sudden_change_repaired
+
         if correct_volume:
             f_na = df2['Volume'].isna()
             if f_na.any():