Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 20 additions & 2 deletions pyrit/score/scorer_evaluation/human_labeled_dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,7 @@ def from_csv(

Objective labels are validated without numeric rounding and must represent
exactly 0 or 1, or a boolean. Harm labels are floats between 0.0 and 1.0.
Only an empty cell counts as missing; text such as "None", "null" or "N/A" is kept as written.

You can optionally include # comment lines at the top of the CSV file to specify
the dataset version and harm definition path; the metadata may appear on any of
Expand Down Expand Up @@ -325,11 +326,28 @@ def from_csv(
)
# Keep objective labels as text until validation, before rounding or underflow can occur.
csv_dtypes = str if metrics_type == MetricsType.OBJECTIVE else text_dtypes
# Only an empty cell is missing. By default pandas also reads cells such as "None", "null",
# "N/A" and "nan" as missing, although they are valid responses, objectives and harm categories.
missing_values = [""]
# Try UTF-8 first, fall back to latin-1 for files with special characters
try:
eval_df = pd.read_csv(csv_path, skiprows=skiprows or None, encoding="utf-8", dtype=csv_dtypes)
eval_df = pd.read_csv(
csv_path,
skiprows=skiprows or None,
encoding="utf-8",
dtype=csv_dtypes,
keep_default_na=False,
na_values=missing_values,
)
except UnicodeDecodeError:
eval_df = pd.read_csv(csv_path, skiprows=skiprows or None, encoding="latin-1", dtype=csv_dtypes)
eval_df = pd.read_csv(
csv_path,
skiprows=skiprows or None,
encoding="latin-1",
dtype=csv_dtypes,
keep_default_na=False,
na_values=missing_values,
)

# Drop rows where every column is NaN (e.g. trailing blank lines in the CSV)
eval_df = eval_df.dropna(how="all")
Expand Down
44 changes: 44 additions & 0 deletions tests/unit/score/test_human_labeled_dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,50 @@ def test_from_csv_preserves_numeric_and_boolean_text(
assert entry.human_scores == [0.5]


@pytest.mark.parametrize("value", ["None", "null", "NULL", "NA", "N/A", "n/a", "nan", "NaN", "<NA>"])
@pytest.mark.parametrize("encoding", ["utf-8", "latin-1"])
@pytest.mark.parametrize("metrics_type", [MetricsType.OBJECTIVE, MetricsType.HARM])
def test_from_csv_preserves_text_that_names_a_missing_value(
tmp_path: Path, value: str, encoding: str, metrics_type: MetricsType
) -> None:
column = "objective" if metrics_type == MetricsType.OBJECTIVE else "harm_category"
label = "1" if metrics_type == MetricsType.OBJECTIVE else "0.5"
csv_path = tmp_path / "text.csv"
csv_path.write_text(
f"assistant_response,{column},human_score,notes\n{value},{value},{label},café\n",
encoding=encoding,
)

dataset = HumanLabeledDataset.from_csv(csv_path=csv_path, metrics_type=metrics_type, version="1.0")
entry = dataset.entries[0]
piece = entry.conversation[0].message_pieces[0]

assert piece.original_value == value
assert piece.converted_value == value
if isinstance(entry, ObjectiveHumanLabeledEntry):
assert entry.objective == value
assert entry.human_scores == [True]
else:
assert isinstance(entry, HarmHumanLabeledEntry)
assert entry.harm_category == value
assert entry.human_scores == [0.5]


@pytest.mark.parametrize("metrics_type", [MetricsType.OBJECTIVE, MetricsType.HARM])
def test_from_csv_rejects_empty_required_cells(tmp_path: Path, metrics_type: MetricsType) -> None:
column = "objective" if metrics_type == MetricsType.OBJECTIVE else "harm_category"
label = "1" if metrics_type == MetricsType.OBJECTIVE else "0.5"
csv_path = tmp_path / "missing.csv"

csv_path.write_text(f"assistant_response,{column},human_score\n,topic,{label}\n", encoding="utf-8")
with pytest.raises(ValueError, match="Column 'assistant_response' contains NaN values"):
HumanLabeledDataset.from_csv(csv_path=csv_path, metrics_type=metrics_type, version="1.0")

csv_path.write_text(f"assistant_response,{column},human_score\nresponse,topic,\n", encoding="utf-8")
with pytest.raises(ValueError, match="All human score cells must be filled"):
HumanLabeledDataset.from_csv(csv_path=csv_path, metrics_type=metrics_type, version="1.0")


@pytest.fixture
def sample_messages():
return [
Expand Down