feat(consolidation): enable observation dedup by default (0.97), skip on Oracle

hindsight-api-slim/hindsight_api/config.py

@@ -814,10 +814,11 @@ def _parse_strategy_boosts(raw: str | None) -> dict[str, str]:
     100  # Max memories per consolidation round (0 = unlimited). Limits how long one bank holds a worker slot.
 )
 DEFAULT_CONSOLIDATION_LLM_BATCH_SIZE = 8  # Facts per LLM call (1 = no batching; >1 = batch mode)
-# Cosine >= this between a newly-created observation and an existing one triggers a focused
-# 1-by-1 LLM "merge or keep" pass (the LLM reads both, so numbers/negation/entities are
-# respected). 1.0 disables it (no obs is ever >=1.0 to a *different* one after the exact guard).
-DEFAULT_CONSOLIDATION_DEDUP_THRESHOLD = 1.0
+# Cosine >= this between a newly-created or freshly-updated observation and an existing one
+# triggers a focused 1-by-1 LLM "merge or keep" pass (the LLM reads both, so numbers/negation/
+# entities are respected). Enabled by default; set to 1.0 to disable. Postgres only — the merge
+# path uses Postgres-only SQL, so consolidation skips it on Oracle regardless of this value.
+DEFAULT_CONSOLIDATION_DEDUP_THRESHOLD = 0.97
 DEFAULT_CONSOLIDATION_LLM_PARALLELISM = (
     4  # Max tag groups consolidated concurrently per op. Locks on overlapping write
     # scopes degrade to sequential automatically; matches retain_max_concurrent.

hindsight-api-slim/hindsight_api/engine/consolidation/consolidator.py

@@ -112,6 +112,19 @@ class _DedupDecision(BaseModel):
 number/quantity, a named entity or language, a negation, or a condition — respond action="keep"."""
 
 
+def _dedup_active(config: Any) -> bool:
+    """Whether create/update semantic dedup runs for this consolidation.
+
+    Enabled when the resolved threshold is < 1.0, EXCEPT on Oracle: the merge path uses
+    Postgres-only SQL (``unnest``/``array_agg``, ``UPDATE ... FROM``), so on Oracle dedup is
+    skipped — it behaves exactly as it did before this feature, regardless of the configured
+    threshold. This is why the feature can ship enabled-by-default without breaking Oracle.
+    """
+    if config is None or getattr(config, "consolidation_dedup_threshold", 1.0) >= 1.0:
+        return False
+    return get_config().database_backend != "oracle"
+
+
 @dataclass
 class _DedupOutcome:
     """Result of probing one observation against its in-scope neighbours.
@@ -1351,7 +1364,7 @@ async def _process_memory_batch(
     # observation into a twin — the create-time guard can't see this). The trace operation/scope is
     # "consolidation_dedup" (routes through the consolidation concurrency bucket via llm_wrapper's
     # "consolidation" prefix; recorded distinctly in llm_requests).
-    dedup_enabled = config is not None and getattr(config, "consolidation_dedup_threshold", 1.0) < 1.0
+    dedup_enabled = _dedup_active(config)
     dedup_llm_config = (
         memory_engine._consolidation_llm_config.with_config(config, bank_id=bank_id, operation="consolidation_dedup")
         if dedup_enabled

hindsight-api-slim/hindsight_api/engine/providers/mock_llm.py

@@ -162,6 +162,12 @@ async def call(
             # Consolidation: produce a single observation from the input facts
             # so the full pipeline (retain → consolidation → observation → recall) works.
             result = self._build_mock_consolidation(messages, response_format)
+        elif scope == "consolidation_dedup" and response_format is not None:
+            # Observation dedup adjudication. Default to "keep" so mock-LLM consolidation never
+            # spuriously merges observations — this preserves the pre-dedup behaviour that
+            # deterministic consolidation tests assert (the generic branch below can't construct
+            # the model because its "action" field is required and has no default).
+            result = response_format(action="keep", reason="mock")
         elif scope == "memory_think":
             # Reflect: return a plausible text answer
             result = "Based on the available information, the answer is related to the context provided."

hindsight-api-slim/tests/test_consolidation_dedup.py

@@ -11,6 +11,7 @@
 from unittest.mock import AsyncMock, patch
 
 from hindsight_api.engine.consolidation.consolidator import (
+    _dedup_active,
     _dedup_reconcile_create,
     _dedup_reconcile_update,
     _DedupDecision,
@@ -219,3 +220,40 @@ async def test_dedup_update_no_twin_above_threshold() -> None:
         await _dedup_reconcile_update(**kwargs)
     llm.call.assert_not_called()
     conn.execute.assert_not_called()
+
+
+# ── dedup activation gate (_dedup_active) ─────────────────────────────────────
+#
+# Enabled by default (threshold < 1.0), but skipped on Oracle because the merge path is
+# Postgres-only — so the feature can ship on-by-default without breaking Oracle.
+
+
+def _gate_cfg(threshold: float):
+    return types.SimpleNamespace(consolidation_dedup_threshold=threshold)
+
+
+def _patch_backend(name: str):
+    return patch(
+        "hindsight_api.engine.consolidation.consolidator.get_config",
+        return_value=types.SimpleNamespace(database_backend=name),
+    )
+
+
+def test_dedup_active_enabled_on_postgres() -> None:
+    with _patch_backend("postgresql"):
+        assert _dedup_active(_gate_cfg(0.97)) is True
+
+
+def test_dedup_active_disabled_when_threshold_is_one() -> None:
+    with _patch_backend("postgresql"):
+        assert _dedup_active(_gate_cfg(1.0)) is False
+
+
+def test_dedup_active_skipped_on_oracle() -> None:
+    # PG-only merge path → dedup is skipped on Oracle even with a sub-1.0 threshold.
+    with _patch_backend("oracle"):
+        assert _dedup_active(_gate_cfg(0.97)) is False
+
+
+def test_dedup_active_none_config() -> None:
+    assert _dedup_active(None) is False

hindsight-docs/docs/developer/configuration.md

@@ -1220,7 +1220,7 @@ Observations are deduplicated, evidence-grounded knowledge consolidated from mul
 | `HINDSIGHT_API_CONSOLIDATION_MAX_MEMORIES_PER_ROUND` | Maximum memories processed per consolidation round. When the limit is reached, the job yields its worker slot and re-queues itself so other banks get fair scheduling. Mental model refreshes only run on the final round. `0` = unlimited. Configurable per bank. | `100` |
 | `HINDSIGHT_API_CONSOLIDATION_MAX_TOKENS` | Max tokens for recall when finding related observations during consolidation | `1024` |
 | `HINDSIGHT_API_CONSOLIDATION_LLM_BATCH_SIZE` | Number of facts sent to the LLM in a single consolidation call. Higher values reduce LLM calls and improve throughput at the cost of larger prompts. Set to `1` to disable batching. Configurable per bank. | `8` |
-| `HINDSIGHT_API_CONSOLIDATION_DEDUP_THRESHOLD` | Cosine similarity at/above which a newly-created or freshly-updated observation is reconciled against an existing near-identical one via a focused 1-by-1 LLM "merge or keep" call (the model reads both texts, so a number/negation/entity difference is respected). Catches near-duplicate observations that weaker consolidation models emit even when shown the twin, as well as duplicates that arise when an update rewrites an observation into a near-twin of another. `1.0` disables it. Postgres only. | `1.0` (disabled) |
+| `HINDSIGHT_API_CONSOLIDATION_DEDUP_THRESHOLD` | Cosine similarity at/above which a newly-created or freshly-updated observation is reconciled against an existing near-identical one via a focused 1-by-1 LLM "merge or keep" call (the model reads both texts, so a number/negation/entity difference is respected). Catches near-duplicate observations that weaker consolidation models emit even when shown the twin, as well as duplicates that arise when an update rewrites an observation into a near-twin of another. Set to `1.0` to disable. Postgres only — consolidation skips reconciliation on Oracle regardless of this value. | `0.97` |
 | `HINDSIGHT_API_CONSOLIDATION_LLM_PARALLELISM` | Maximum number of tag groups consolidated concurrently within one consolidation op. Each group acquires per-scope locks before processing, so groups whose write scopes overlap (e.g. under `per_tag` / `all_combinations` / explicit-list `observation_scopes`) automatically serialise on the overlapping scopes — actual concurrency may be lower than this cap when scopes contend. Set to `1` for fully sequential behaviour. Higher values raise peak LLM QPS and connection-pool usage during consolidation proportionally — tune down if your LLM provider rate-limits tightly or your DB pool is small. Configurable per bank. | `4` |
 | `HINDSIGHT_API_CONSOLIDATION_RECALL_BUDGET` | Budget level for the recall pass inside consolidation (`low`, `mid`, `high`). Lower budgets fetch fewer candidate rows, reducing peak memory usage on large banks. | `low` |
 | `HINDSIGHT_API_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS` | Total token budget for source facts included with observations in the consolidation prompt. `-1` = unlimited. Configurable per bank. | `4096` |

hindsight-docs/docs/developer/observations.mdx

@@ -58,7 +58,7 @@ Consolidation can still produce two observations that say the same thing in slig
 
 When enabled, Hindsight reconciles them automatically. Whenever an observation is created **or** updated, it is compared against the existing observations it most closely resembles. If one is highly similar, a focused check decides whether to **merge** them into a single belief (folding both sets of supporting evidence together) or **keep** them separate. Because the check reads the full text of both, observations that differ in a meaningful detail — a number, a negation, a named entity or language — are correctly kept apart rather than collapsed.
 
-This is controlled by the [`HINDSIGHT_API_CONSOLIDATION_DEDUP_THRESHOLD`](/developer/configuration#observations) setting: the cosine similarity at or above which two observations are reconciled. It is **disabled by default** (`1.0`) and is available on PostgreSQL deployments. Lower it (e.g. `0.97`) to enable reconciliation; a lower value reconciles more aggressively.
+This is controlled by the [`HINDSIGHT_API_CONSOLIDATION_DEDUP_THRESHOLD`](/developer/configuration#observations) setting: the cosine similarity at or above which two observations are reconciled. It is **enabled by default** (`0.97`); a lower value reconciles more aggressively, and `1.0` disables it. Reconciliation runs on PostgreSQL deployments only — it is skipped on Oracle regardless of the threshold.
 
 ### Disabling Auto-Consolidation
 

skills/hindsight-docs/references/developer/configuration.md

@@ -1220,7 +1220,7 @@ Observations are deduplicated, evidence-grounded knowledge consolidated from mul
 | `HINDSIGHT_API_CONSOLIDATION_MAX_MEMORIES_PER_ROUND` | Maximum memories processed per consolidation round. When the limit is reached, the job yields its worker slot and re-queues itself so other banks get fair scheduling. Mental model refreshes only run on the final round. `0` = unlimited. Configurable per bank. | `100` |
 | `HINDSIGHT_API_CONSOLIDATION_MAX_TOKENS` | Max tokens for recall when finding related observations during consolidation | `1024` |
 | `HINDSIGHT_API_CONSOLIDATION_LLM_BATCH_SIZE` | Number of facts sent to the LLM in a single consolidation call. Higher values reduce LLM calls and improve throughput at the cost of larger prompts. Set to `1` to disable batching. Configurable per bank. | `8` |
-| `HINDSIGHT_API_CONSOLIDATION_DEDUP_THRESHOLD` | Cosine similarity at/above which a newly-created or freshly-updated observation is reconciled against an existing near-identical one via a focused 1-by-1 LLM "merge or keep" call (the model reads both texts, so a number/negation/entity difference is respected). Catches near-duplicate observations that weaker consolidation models emit even when shown the twin, as well as duplicates that arise when an update rewrites an observation into a near-twin of another. `1.0` disables it. Postgres only. | `1.0` (disabled) |
+| `HINDSIGHT_API_CONSOLIDATION_DEDUP_THRESHOLD` | Cosine similarity at/above which a newly-created or freshly-updated observation is reconciled against an existing near-identical one via a focused 1-by-1 LLM "merge or keep" call (the model reads both texts, so a number/negation/entity difference is respected). Catches near-duplicate observations that weaker consolidation models emit even when shown the twin, as well as duplicates that arise when an update rewrites an observation into a near-twin of another. Set to `1.0` to disable. Postgres only — consolidation skips reconciliation on Oracle regardless of this value. | `0.97` |
 | `HINDSIGHT_API_CONSOLIDATION_LLM_PARALLELISM` | Maximum number of tag groups consolidated concurrently within one consolidation op. Each group acquires per-scope locks before processing, so groups whose write scopes overlap (e.g. under `per_tag` / `all_combinations` / explicit-list `observation_scopes`) automatically serialise on the overlapping scopes — actual concurrency may be lower than this cap when scopes contend. Set to `1` for fully sequential behaviour. Higher values raise peak LLM QPS and connection-pool usage during consolidation proportionally — tune down if your LLM provider rate-limits tightly or your DB pool is small. Configurable per bank. | `4` |
 | `HINDSIGHT_API_CONSOLIDATION_RECALL_BUDGET` | Budget level for the recall pass inside consolidation (`low`, `mid`, `high`). Lower budgets fetch fewer candidate rows, reducing peak memory usage on large banks. | `low` |
 | `HINDSIGHT_API_CONSOLIDATION_SOURCE_FACTS_MAX_TOKENS` | Total token budget for source facts included with observations in the consolidation prompt. `-1` = unlimited. Configurable per bank. | `4096` |

skills/hindsight-docs/references/developer/observations.md

@@ -52,7 +52,7 @@ Consolidation can still produce two observations that say the same thing in slig
 
 When enabled, Hindsight reconciles them automatically. Whenever an observation is created **or** updated, it is compared against the existing observations it most closely resembles. If one is highly similar, a focused check decides whether to **merge** them into a single belief (folding both sets of supporting evidence together) or **keep** them separate. Because the check reads the full text of both, observations that differ in a meaningful detail — a number, a negation, a named entity or language — are correctly kept apart rather than collapsed.
 
-This is controlled by the [`HINDSIGHT_API_CONSOLIDATION_DEDUP_THRESHOLD`](configuration.md#observations) setting: the cosine similarity at or above which two observations are reconciled. It is **disabled by default** (`1.0`) and is available on PostgreSQL deployments. Lower it (e.g. `0.97`) to enable reconciliation; a lower value reconciles more aggressively.
+This is controlled by the [`HINDSIGHT_API_CONSOLIDATION_DEDUP_THRESHOLD`](configuration.md#observations) setting: the cosine similarity at or above which two observations are reconciled. It is **enabled by default** (`0.97`); a lower value reconciles more aggressively, and `1.0` disables it. Reconciliation runs on PostgreSQL deployments only — it is skipped on Oracle regardless of the threshold.
 
 ### Disabling Auto-Consolidation