diff --git a/README.md b/README.md index 7b94103..b964111 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ texts = load_dataset("ag_news", split="train")["text"] semhash = SemHash.from_records(records=texts) # Deduplicate the texts -deduplicated_texts = semhash.self_deduplicate().deduplicated +deduplicated_texts = semhash.self_deduplicate().selected ``` Or, deduplicate across two datasets with the following code (e.g., eliminating train/test leakage): @@ -78,7 +78,7 @@ test_texts = load_dataset("ag_news", split="test")["text"] semhash = SemHash.from_records(records=train_texts) # Deduplicate the test data against the training data, optionally with a specific threshold -deduplicated_test_texts = semhash.deduplicate(records=test_texts, threshold=0.9).deduplicated +deduplicated_test_texts = semhash.deduplicate(records=test_texts, threshold=0.9).selected ``` Or, deduplicate multi-column datasets with the following code (e.g., deduplicating a QA dataset): @@ -97,7 +97,7 @@ records = [dict(row) for row in dataset] semhash = SemHash.from_records(records=records, columns=["question", "context"]) # Deduplicate the records -deduplicated_records = semhash.self_deduplicate().deduplicated +deduplicated_records = semhash.self_deduplicate().selected ``` The `deduplicate` and `self_deduplicate` functions return a [DeduplicationResult](https://github.com/MinishLab/semhash/blob/main/semhash/datamodels.py#L30). This object stores the deduplicated corpus, a set of duplicate object (along with the objects that caused duplication), and several useful functions to further inspect the deduplication result. @@ -185,7 +185,7 @@ records = [dict(row) for row in dataset] semhash = SemHash.from_records(records=records, columns=["question", "context"]) # Deduplicate the records -deduplicated_records = semhash.self_deduplicate().deduplicated +deduplicated_records = semhash.self_deduplicate().selected ``` @@ -210,9 +210,9 @@ semhash = SemHash.from_records(records=texts) deduplication_result = semhash.self_deduplicate() # Check the deduplicated texts -deduplication_result.deduplicated +deduplication_result.selected # Check the duplicates -deduplication_result.duplicates +deduplication_result.filtered # See what percentage of the texts were duplicates deduplication_result.duplicate_ratio # See what percentage of the texts were exact duplicates @@ -294,7 +294,7 @@ dataframe = dataframe.to_dict(orient="records") semhash = SemHash.from_records(records=dataframe, columns=["text"]) # Deduplicate the texts -deduplicated_records = semhash.self_deduplicate().deduplicated +deduplicated_records = semhash.self_deduplicate().selected # Convert the deduplicated records back to a pandas dataframe deduplicated_dataframe = pd.DataFrame(deduplicated_records) @@ -337,7 +337,7 @@ texts = load_dataset("ag_news", split="train")["text"] semhash = SemHash.from_records(records=texts) # Deduplicate the records -deduplicated_records = semhash.self_deduplicate().deduplicated +deduplicated_records = semhash.self_deduplicate().selected # Filter the records based on entropy score filtered_records = semhash.filter_by_entropy( diff --git a/benchmarks/run_benchmarks.py b/benchmarks/run_benchmarks.py index 0178628..128b01b 100644 --- a/benchmarks/run_benchmarks.py +++ b/benchmarks/run_benchmarks.py @@ -55,7 +55,7 @@ def main() -> None: # noqa: C901 train_only_dedup_time = train_only_end - train_only_start original_train_size = len(train_records) - dedup_train_size = len(deduplicated_train.deduplicated) + dedup_train_size = len(deduplicated_train.selected) percent_removed_train = deduplicated_train.duplicate_ratio * 100 train_dedup_results.append( @@ -88,7 +88,7 @@ def main() -> None: # noqa: C901 train_test_end = perf_counter() train_test_dedup_time = train_test_end - train_test_start original_test_size = len(test_records) - deduped_test_size = len(deduplicated_test.deduplicated) + deduped_test_size = len(deduplicated_test.selected) percent_removed_test = deduplicated_test.duplicate_ratio * 100 train_test_dedup_results.append( diff --git a/semhash/datamodels.py b/semhash/datamodels.py index e17096f..d2c93ca 100644 --- a/semhash/datamodels.py +++ b/semhash/datamodels.py @@ -1,3 +1,4 @@ +import warnings from dataclasses import dataclass, field from typing import Generic, Optional, TypeVar @@ -33,28 +34,50 @@ class DeduplicationResult(Generic[Record]): Attributes ---------- - deduplicated: List of deduplicated records after removing duplicates. - duplicates: List of DuplicateRecord objects containing details about duplicates of an original record. threshold: The similarity threshold used for deduplication. + selected: List of deduplicated records after removing duplicates. + filtered: List of DuplicateRecord objects containing details about duplicates of an original record. + deduplicated: Deprecated, use selected instead. + duplicates: Deprecated, use filtered instead. """ - deduplicated: list[Record] - duplicates: list[DuplicateRecord] - threshold: float + selected: list[Record] = field(default_factory=list) + filtered: list[DuplicateRecord] = field(default_factory=list) + threshold: float = field(default=0.9) + deduplicated: list[Record] = field(default_factory=list) # Deprecated + duplicates: list[DuplicateRecord] = field(default_factory=list) # Deprecated + + def __post_init__(self) -> None: + """Initialize deprecated fields and warn about deprecation.""" + if self.deduplicated or self.duplicates: + warnings.warn( + "'deduplicated' and 'duplicates' fields are deprecated and will be removed in a `semhash==0.3.0` release. Use 'selected' and 'filtered' instead.", + DeprecationWarning, + stacklevel=2, + ) + + if not self.selected and self.deduplicated: + self.selected = self.deduplicated + if not self.filtered and self.duplicates: + self.filtered = self.duplicates + if not self.deduplicated: + self.deduplicated = self.selected + if not self.duplicates: + self.duplicates = self.filtered @property def duplicate_ratio(self) -> float: """Return the percentage of records dropped.""" - if denom := len(self.deduplicated) + len(self.duplicates): - return 1.0 - len(self.deduplicated) / denom + if denom := len(self.selected) + len(self.filtered): + return 1.0 - len(self.selected) / denom return 0.0 @property def exact_duplicate_ratio(self) -> float: """Return the percentage of records dropped due to an exact match.""" - if denom := len(self.deduplicated) + len(self.duplicates): - return len([dup for dup in self.duplicates if dup.exact]) / denom + if denom := len(self.selected) + len(self.filtered): + return len([dup for dup in self.filtered if dup.exact]) / denom return 0.0 def get_least_similar_from_duplicates(self, n: int = 1) -> list[tuple[Record, Record, float]]: @@ -64,7 +87,7 @@ def get_least_similar_from_duplicates(self, n: int = 1) -> list[tuple[Record, Re :param n: The number of least similar pairs to return. :return: A list of tuples consisting of (original_record, duplicate_record, score). """ - all_pairs = [(dup.record, d, score) for dup in self.duplicates for d, score in dup.duplicates] + all_pairs = [(dup.record, d, score) for dup in self.filtered for d, score in dup.duplicates] sorted_pairs = sorted(all_pairs, key=lambda x: x[2]) # Sort by score return sorted_pairs[:n] @@ -72,11 +95,11 @@ def rethreshold(self, threshold: float) -> None: """Rethreshold the duplicates.""" if self.threshold > threshold: raise ValueError("Threshold is smaller than the given value.") - for dup in self.duplicates: + for dup in self.filtered: dup._rethreshold(threshold) if not dup.duplicates: - self.duplicates.remove(dup) - self.deduplicated.append(dup.record) + self.filtered.remove(dup) + self.selected.append(dup.record) self.threshold = threshold diff --git a/semhash/records.py b/semhash/records.py index 2dcbc42..69fd427 100644 --- a/semhash/records.py +++ b/semhash/records.py @@ -26,7 +26,7 @@ def dict_to_string(record: dict[str, str], columns: Sequence[str]) -> str: def map_deduplication_result_to_strings(result: DeduplicationResult, columns: Sequence[str]) -> DeduplicationResult: """Convert the record and duplicates in each DuplicateRecord back to strings if self.was_string is True.""" - deduplicated_str = [dict_to_string(r, columns) for r in result.deduplicated] + deduplicated_str = [dict_to_string(r, columns) for r in result.selected] mapped = [] for dup_rec in result.duplicates: record_as_str = dict_to_string(dup_rec.record, columns) @@ -38,7 +38,7 @@ def map_deduplication_result_to_strings(result: DeduplicationResult, columns: Se exact=dup_rec.exact, ) ) - return DeduplicationResult(deduplicated=deduplicated_str, duplicates=mapped, threshold=result.threshold) + return DeduplicationResult(selected=deduplicated_str, filtered=mapped, threshold=result.threshold) def add_scores_to_records(records: list[dict[str, str]]) -> list[tuple[dict[str, str], float]]: diff --git a/tests/test_datamodels.py b/tests/test_datamodels.py index a67e726..433ba87 100644 --- a/tests/test_datamodels.py +++ b/tests/test_datamodels.py @@ -1,5 +1,7 @@ import pytest +import semhash +import semhash.version from semhash.datamodels import DeduplicationResult, DuplicateRecord @@ -77,8 +79,8 @@ def test_rethreshold_deduplication_result() -> None: 0.8, ) d.rethreshold(0.85) - assert d.duplicates == [DuplicateRecord("d", False, [("x", 0.9)])] - assert d.deduplicated == ["a", "b", "c", "e"] + assert d.filtered == [DuplicateRecord("d", False, [("x", 0.9)])] + assert d.selected == ["a", "b", "c", "e"] def test_rethreshold_exception() -> None: @@ -93,3 +95,24 @@ def test_rethreshold_exception() -> None: ) with pytest.raises(ValueError): d.rethreshold(0.6) + + +def test_deprecation_deduplicated_duplicates() -> None: + """Test deprecation warnings for deduplicated and duplicates fields.""" + if semhash.version.__version__ < "0.3.0": + with pytest.warns(DeprecationWarning): + d = DeduplicationResult( + deduplicated=["a", "b", "c"], + duplicates=[ + DuplicateRecord("d", False, [("x", 0.9), ("y", 0.8)]), + DuplicateRecord("e", False, [("z", 0.8)]), + ], + threshold=0.8, + ) + else: + raise ValueError("deprecate `deduplicated` and `duplicates` fields in `DeduplicationResult`") + assert d.selected == ["a", "b", "c"] + assert d.filtered == [ + DuplicateRecord("d", False, [("x", 0.9), ("y", 0.8)]), + DuplicateRecord("e", False, [("z", 0.8)]), + ] diff --git a/tests/test_semhash.py b/tests/test_semhash.py index 28c1dd0..974904f 100644 --- a/tests/test_semhash.py +++ b/tests/test_semhash.py @@ -13,7 +13,7 @@ def test_single_dataset_deduplication(use_ann: bool, model: Encoder) -> None: "Ganondorf has invaded Hyrule!", ] semhash = SemHash.from_records(records=texts, use_ann=use_ann, model=model) - deduplicated_texts = semhash.self_deduplicate().deduplicated + deduplicated_texts = semhash.self_deduplicate().selected assert deduplicated_texts == texts @@ -24,7 +24,7 @@ def test_single_dataset_deduplication(use_ann: bool, model: Encoder) -> None: "It's not safe to go alone!", # Semantically similar ] semhash = SemHash.from_records(records=texts, use_ann=use_ann, model=model) - deduplicated_texts = semhash.self_deduplicate().deduplicated + deduplicated_texts = semhash.self_deduplicate().selected assert deduplicated_texts == ["It's dangerous to go alone!"] @@ -42,7 +42,7 @@ def test_multi_dataset_deduplication(use_ann: bool, model: Encoder) -> None: "Ganon is the king of thieves.", ] semhash = SemHash.from_records(texts1, columns=None, use_ann=use_ann, model=model) - deduplicated_texts = semhash.deduplicate(texts2).deduplicated + deduplicated_texts = semhash.deduplicate(texts2).selected assert deduplicated_texts == texts2 # With duplicates @@ -51,7 +51,7 @@ def test_multi_dataset_deduplication(use_ann: bool, model: Encoder) -> None: "It's risky to go alone!", # Semantically similar "Ganondorf has attacked Hyrule!", # Semantically similar ] - deduplicated_texts = semhash.deduplicate(texts2).deduplicated + deduplicated_texts = semhash.deduplicate(texts2).selected assert deduplicated_texts == [] @@ -75,7 +75,7 @@ def test_single_dataset_deduplication_multicolumn(use_ann: bool, model: Encoder) ) deduplicated = semhash.self_deduplicate() - assert deduplicated.deduplicated == [ + assert deduplicated.selected == [ {"question": "What is the hero's name?", "context": "The hero is Link", "answer": "Link"}, {"question": "Who is the princess?", "context": "The princess is Zelda", "answer": "Zelda"}, ] @@ -102,7 +102,7 @@ def test_multi_dataset_deduplication_multicolumn(use_ann: bool, model: Encoder) use_ann=use_ann, model=model, ) - deduplicated = semhash.deduplicate(test_records).deduplicated + deduplicated = semhash.deduplicate(test_records).selected assert deduplicated == [ {"question": "What is the villain's name?", "context": "The villain is Ganon", "answer": "Ganon"} ] @@ -132,10 +132,10 @@ def test_deduplicate_with_only_exact_duplicates(use_ann: bool, model: Encoder) - ] semhash = SemHash.from_records(texts1, use_ann=use_ann, model=model) deduplicated = semhash.self_deduplicate() - assert deduplicated.deduplicated == ["It's dangerous to go alone!"] + assert deduplicated.selected == ["It's dangerous to go alone!"] deduplicated = semhash.deduplicate(texts2) - assert deduplicated.deduplicated == [] + assert deduplicated.selected == [] def test_filter_by_entropy(use_ann: bool, model: Encoder) -> None: