Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions pyrit/converter/insert_punctuation_converter.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
import re
import string

import regex

from pyrit.converter.converter import Converter, ConverterResult
from pyrit.models import ComponentIdentifier, PromptDataType

Expand Down Expand Up @@ -127,8 +129,9 @@ def _insert_punctuation(self, prompt: str, punctuation_list: list[str]) -> str:
Returns:
str: The modified prompt with inserted punctuation from helper method.
"""
# Words list contains single spaces, single word without punctuations, single punctuations
words = re.findall(r"\w+|[^\w\s]|\s", prompt)
# Words list contains single spaces, single word without punctuations, single punctuations.
# Combining marks (accents, vowel signs) are not \w but belong to the word they follow.
words = regex.findall(r"\w[\w\p{M}]*|[^\w\s]|\s", prompt)
# Maintains indices for actual "words", i.e. letters and numbers not divided by punctuations
word_indices = [i for i in range(len(words)) if not re.match(r"\W", words[i])]
# Calculate the number of insertions
Expand Down
11 changes: 11 additions & 0 deletions tests/unit/converter/test_insert_punctuation_converter.py
Original file line number Diff line number Diff line change
Expand Up @@ -107,3 +107,14 @@ async def test_insert_within_words_without_words_async(prompt: str) -> None:

assert result.output_text == f"!{prompt}"
assert result.output_type == "text"


@pytest.mark.parametrize("prompt", ["café au lait", "नमस्ते दुनिया"])
@pytest.mark.parametrize("seed", range(5))
async def test_insert_between_words_keeps_combining_marks_with_their_word_async(*, prompt: str, seed: int) -> None:
converter = InsertPunctuationConverter(word_swap_ratio=1.0, seed=seed)
result = await converter.convert_async(prompt=prompt, punctuation_list=["!"])

# Every word gets exactly one "!" before or after it, never inside it.
for original, converted in zip(prompt.split(" "), result.output_text.split(" "), strict=True):
assert converted in (f"!{original}", f"{original}!")
Loading