fix: gli_guard and wild_guard both work regardless of transformers major version (#222)

* fix(gli_guard): tolerate list-shaped extra_special_tokens on transformers<5

fastino/gliguard-LLMGuardrails-300M ships tokenizer_config.json's
extra_special_tokens as a list. transformers<5 passes that straight to
_set_model_specific_special_tokens, which assumes a dict and crashes with
AttributeError: 'list' object has no attribute 'keys'. transformers>=5 added
an isinstance(list, tuple) branch that handles this natively.

Shim the same handling for <5, scoped to the one GLiNER2.from_pretrained
call. gliner2's own processor hardcodes and re-injects its special-token
strings by value, never reading these dict keys, so synthesized key names
are safe. Verified against real transformers 4.57.6 and 5.8.0.

Part of #221.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

* fix(deps): declare sentencepiece/protobuf in the huggingface extra

allenai/wildguard ships only a raw SentencePiece tokenizer.model (no fast
tokenizer.json). Without sentencepiece + protobuf installed, transformers
can't extract it and misdetects the file as tiktoken-formatted, failing with
"'tiktoken' is required to read a 'tiktoken' file" on every transformers
major version, not just one. Both packages were previously only present
transitively via the gliner extra, so installing huggingface alone left
WildGuard broken.

Verified by reproducing the exact failure (uninstalling both packages) and
the fix (reinstalling them) against real transformers 5.8.0, and separately
against 5.14.1 in an isolated venv — no version gate is needed, this was
never actually a transformers-major-version conflict.

Part of #221.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

* fix(gli_guard): address Copilot review — thread-safety and test brittleness

Serialize _tolerate_list_extra_special_tokens with a module-level lock: it
mutates a class-wide transformers method for its duration, so concurrent
GliGuard() constructions on transformers<5 must not interleave patch/restore.

Rewrite the shim's unit test to simulate transformers<5's dict-only
implementation with a controlled fake instead of relying on the real
installed transformers' _set_model_specific_special_tokens raising
AttributeError on a list — that's an internal detail that could change
upstream independent of the shim's own correctness.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>

Author: dni138

pyproject.toml

@@ -46,7 +46,12 @@ huggingface = [
   "huggingface-hub>=0.33.4",
   "transformers>=4.53.2",
   "torch>=2.7.1",
-  "hf-xet>=1.1.5"
+  "hf-xet>=1.1.5",
+  # Needed by transformers to convert SentencePiece-only tokenizers (e.g. allenai/wildguard,
+  # which ships no fast tokenizer.json) into a usable tokenizer; without these, transformers
+  # misdetects the file as tiktoken-formatted and fails on every transformers major version.
+  "sentencepiece",
+  "protobuf"
 ]
 
 azure-content-safety = [

src/any_guardrail/guardrails/gli_guard/gli_guard.py

@@ -1,4 +1,7 @@
+import threading
 import time
+from collections.abc import Iterator
+from contextlib import contextmanager
 from typing import Any, ClassVar
 
 from any_guardrail.base import GuardrailName
@@ -59,6 +62,51 @@
 }
 
 
+def _transformers_major() -> int:
+    """Return the installed transformers major version (0 if transformers isn't importable)."""
+    try:
+        from transformers import __version__ as transformers_version
+    except ImportError:
+        return 0
+    return int(transformers_version.split(".")[0])
+
+
+# _tolerate_list_extra_special_tokens mutates a class-wide transformers method for its
+# duration, so concurrent GliGuard() constructions on transformers<5 must not interleave.
+_PATCH_LOCK = threading.Lock()
+
+
+@contextmanager
+def _tolerate_list_extra_special_tokens() -> Iterator[None]:
+    """Backport transformers>=5's list/tuple handling for ``extra_special_tokens`` onto <5.
+
+    ``fastino/gliguard-LLMGuardrails-300M`` ships ``extra_special_tokens`` in
+    ``tokenizer_config.json`` as a JSON list. transformers<5's
+    ``PreTrainedTokenizerBase.__init__`` passes that value straight to
+    ``_set_model_specific_special_tokens``, which assumes a dict and calls ``.keys()``
+    on it, raising ``AttributeError: 'list' object has no attribute 'keys'``.
+    transformers>=5 added an ``isinstance(value, (list, tuple))`` branch upstream that
+    handles this natively, so this shim is only needed below that version. ``gliner2``'s
+    own processor hardcodes and re-injects its special-token strings by value — it never
+    reads these dict keys — so any synthesized key names are safe here.
+    """
+    from transformers import PreTrainedTokenizerBase
+
+    with _PATCH_LOCK:
+        original = PreTrainedTokenizerBase._set_model_specific_special_tokens
+
+        def _patched(self: Any, special_tokens: Any) -> None:
+            if isinstance(special_tokens, (list, tuple)):
+                special_tokens = {f"extra_special_token_{i}": tok for i, tok in enumerate(special_tokens)}
+            original(self, special_tokens)
+
+        PreTrainedTokenizerBase._set_model_specific_special_tokens = _patched  # type: ignore[method-assign]
+        try:
+            yield
+        finally:
+            PreTrainedTokenizerBase._set_model_specific_special_tokens = original  # type: ignore[method-assign]
+
+
 class GliGuard(Guardrail):
     """Schema-driven safety, toxicity, jailbreak, and refusal detector.
 
@@ -131,7 +179,11 @@ def __init__(self, model_id: str | None = None, threshold: float = 0.5) -> None:
             raise ImportError(msg) from MISSING_PACKAGES_ERROR
         self.model_id = default(model_id, self.SUPPORTED_MODELS)
         self.threshold = threshold
-        self.model = GLiNER2.from_pretrained(self.model_id)
+        if _transformers_major() < 5:
+            with _tolerate_list_extra_special_tokens():
+                self.model = GLiNER2.from_pretrained(self.model_id)
+        else:
+            self.model = GLiNER2.from_pretrained(self.model_id)
 
     def validate(self, input_text: str, **kwargs: Any) -> GuardrailOutput:
         """Classify ``input_text`` across the safety / toxicity / jailbreak / refusal schema.

tests/unit/test_unit_special_guardrails.py

@@ -5,9 +5,10 @@
 from unittest.mock import MagicMock, patch
 
 import pytest
+from transformers import PreTrainedTokenizerBase
 
 from any_guardrail.guardrails.flowjudge.flowjudge import MISSING_PACKAGES_ERROR, Flowjudge
-from any_guardrail.guardrails.gli_guard.gli_guard import GliGuard
+from any_guardrail.guardrails.gli_guard.gli_guard import GliGuard, _tolerate_list_extra_special_tokens
 from any_guardrail.guardrails.lettuce_detect.lettuce_detect import LettuceDetect
 
 # --- LettuceDetect -------------------------------------------------------------
@@ -93,6 +94,82 @@ def test_gli_guard_safe() -> None:
     assert guard.validate("what's the weather?").valid is True
 
 
+def _legacy_set_model_specific_special_tokens(self: Any, special_tokens: Any) -> None:
+    """Mirror transformers<5's dict-only implementation, independent of the installed version.
+
+    Real transformers (both <5 and >=5) implements this by calling ``.keys()``/``.items()``
+    unconditionally, so a raw list crashes with ``AttributeError``. Pinning that behavior here
+    (rather than calling the real, installed method) keeps this test from becoming brittle to
+    upstream changes in transformers' own implementation.
+    """
+    self.SPECIAL_TOKENS_ATTRIBUTES = self.SPECIAL_TOKENS_ATTRIBUTES + list(special_tokens.keys())
+    for key, value in special_tokens.items():
+        self._special_tokens_map[key] = value
+
+
+def test_tolerate_list_extra_special_tokens_converts_list_to_dict() -> None:
+    """The shim backports transformers>=5's list/tuple handling onto a transformers<5-style callable."""
+    fake: Any = SimpleNamespace(SPECIAL_TOKENS_ATTRIBUTES=[], _special_tokens_map={})
+    with patch.object(
+        PreTrainedTokenizerBase, "_set_model_specific_special_tokens", _legacy_set_model_specific_special_tokens
+    ):
+        with pytest.raises(AttributeError):
+            # Unpatched, the raw list crashes exactly like transformers<5 does.
+            PreTrainedTokenizerBase._set_model_specific_special_tokens(fake, ["[SEP_STRUCT]", "[SEP_TEXT]"])  # type: ignore[arg-type]
+
+        with _tolerate_list_extra_special_tokens():
+            PreTrainedTokenizerBase._set_model_specific_special_tokens(fake, ["[SEP_STRUCT]", "[SEP_TEXT]"])  # type: ignore[arg-type]
+
+        assert PreTrainedTokenizerBase._set_model_specific_special_tokens is _legacy_set_model_specific_special_tokens
+
+    assert fake._special_tokens_map == {"extra_special_token_0": "[SEP_STRUCT]", "extra_special_token_1": "[SEP_TEXT]"}
+
+
+def test_gli_guard_transformers_5_skips_patch() -> None:
+    """On transformers>=5, GLiNER2.from_pretrained is called directly, unpatched."""
+    with (
+        patch("any_guardrail.guardrails.gli_guard.gli_guard._transformers_major", return_value=5),
+        patch("any_guardrail.guardrails.gli_guard.gli_guard._tolerate_list_extra_special_tokens") as mock_shim,
+        patch("any_guardrail.guardrails.gli_guard.gli_guard.GLiNER2") as mock_gliner2,
+    ):
+        mock_gliner2.from_pretrained.return_value = MagicMock()
+        GliGuard()
+        mock_shim.assert_not_called()
+        mock_gliner2.from_pretrained.assert_called_once_with("fastino/gliguard-LLMGuardrails-300M")
+
+
+def test_gli_guard_transformers_below_5_applies_and_restores_patch() -> None:
+    """On transformers<5, the shim wraps the load and is restored afterward."""
+    original = PreTrainedTokenizerBase._set_model_specific_special_tokens
+
+    def _assert_patched_during_call(_model_id: str) -> MagicMock:
+        assert PreTrainedTokenizerBase._set_model_specific_special_tokens is not original
+        return MagicMock()
+
+    with (
+        patch("any_guardrail.guardrails.gli_guard.gli_guard._transformers_major", return_value=4),
+        patch("any_guardrail.guardrails.gli_guard.gli_guard.GLiNER2") as mock_gliner2,
+    ):
+        mock_gliner2.from_pretrained.side_effect = _assert_patched_during_call
+        GliGuard()
+
+    assert PreTrainedTokenizerBase._set_model_specific_special_tokens is original
+
+
+def test_gli_guard_patch_restored_on_load_failure() -> None:
+    """The shim is restored via `finally` even when GLiNER2.from_pretrained raises."""
+    original = PreTrainedTokenizerBase._set_model_specific_special_tokens
+    with (
+        patch("any_guardrail.guardrails.gli_guard.gli_guard._transformers_major", return_value=4),
+        patch("any_guardrail.guardrails.gli_guard.gli_guard.GLiNER2") as mock_gliner2,
+    ):
+        mock_gliner2.from_pretrained.side_effect = RuntimeError("boom")
+        with pytest.raises(RuntimeError, match="boom"):
+            GliGuard()
+
+    assert PreTrainedTokenizerBase._set_model_specific_special_tokens is original
+
+
 # --- FlowJudge new init paths --------------------------------------------------
 
 flowjudge_available = pytest.mark.skipif(MISSING_PACKAGES_ERROR is not None, reason="flow-judge not installed")