Gmail-style advanced search for dandiset listing

dandiapi/api/services/search/__init__.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+from dandiapi.api.services.search.parser import SearchSyntaxError, parse_search
+
+# `parse_search` is the only entry point used outside this package; the view
+# layer catches `SearchSyntaxError` and surfaces it as a 400. ParsedSearch /
+# OPERATOR_KEYS are internal — import from .parser directly if you need them.
+__all__ = ['SearchSyntaxError', 'parse_search']

dandiapi/api/services/search/filters.py

@@ -0,0 +1,189 @@
+"""Translate a ParsedSearch into Django ORM filters against the Dandiset queryset."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+import re
+from typing import TYPE_CHECKING
+
+from django.db.models import OuterRef, Subquery
+
+from dandiapi.api.models import Version
+from dandiapi.api.services.search.parser import SearchSyntaxError
+from dandiapi.search.models import AssetSearch
+
+if TYPE_CHECKING:
+    from django.contrib.auth.models import AnonymousUser, User
+    from django.db.models import QuerySet
+
+    from dandiapi.api.models import Dandiset
+    from dandiapi.api.services.search.parser import ParsedSearch
+
+# Aliases for the file-type operator: short name → MIME prefix matched with
+# istartswith. Keep in sync with DandisetSearchQueryParameterSerializer.
+_FILE_TYPE_ALIASES = {
+    'nwb': 'application/x-nwb',
+    'image': 'image/',
+    'text': 'text/',
+    'video': 'video/',
+}
+
+_DATE_OPS = frozenset(
+    {
+        'created_before',
+        'created_after',
+        'modified_before',
+        'modified_after',
+        'published_before',
+        'published_after',
+    }
+)
+_ASSET_OPS = frozenset({'species', 'approach', 'technique', 'standard', 'file_type'})
+
+
+def _annotate_latest_version_modified(queryset):
+    latest_version = Version.objects.filter(dandiset=OuterRef('pk')).order_by('-created')[:1]
+    return queryset.annotate(
+        _search_latest_version_modified=Subquery(latest_version.values('modified'))
+    )
+
+
+def _annotate_latest_published_created(queryset):
+    latest_published = (
+        Version.objects.filter(dandiset=OuterRef('pk'))
+        .exclude(version='draft')
+        .order_by('-created')[:1]
+    )
+    return queryset.annotate(
+        _search_latest_published_created=Subquery(latest_published.values('created'))
+    )
+
+
+# Each entry maps an operator to a Postgres jsonpath that selects the names
+# we want to match against. `[*]` wildcards mean we match if ANY element of
+# the array satisfies the predicate — important for assets that list
+# multiple species, approaches, etc. Paths MUST be trusted constants:
+# they're interpolated into the SQL.
+_NAME_PATH_OPS = {
+    'species': '$.wasAttributedTo[*].species.name',
+    'approach': '$.approach[*].name',
+    'technique': '$.measurementTechnique[*].name',
+    'standard': '$.dataStandard[*].name',
+}
+
+
+def _jsonpath_name_match(path: str, value: str) -> tuple[str, list[str]]:
+    """Build a parameterized Postgres `jsonb_path_exists` predicate.
+
+    Matches `value` case-insensitively as a substring against any node
+    selected by `path`. `path` MUST come from a trusted allowlist; `value`
+    is parameterized and regex-escaped.
+    """
+    # No table prefix on `asset_metadata`: Django may alias the AssetSearch
+    # table (e.g. inside a subquery), and qualifying the column would break
+    # those queries. The unqualified column is unambiguous in our usage.
+    where = (
+        'jsonb_path_exists(asset_metadata, '
+        f"('{path} ? (@ like_regex ' "
+        '|| to_jsonb(%s::text)::text || '
+        '\' flag "i")\')::jsonpath)'
+    )
+    return where, [re.escape(value)]
+
+
+def _apply_asset_filter(queryset, operator: str, value: str):
+    """Apply one parsed asset operator to an AssetSearch queryset."""
+    if operator in _NAME_PATH_OPS:
+        where, params = _jsonpath_name_match(_NAME_PATH_OPS[operator], value)
+        # `where` interpolates only an allowlisted jsonpath; the user value
+        # is bound via params (and re-escaped against regex injection).
+        return queryset.extra(where=[where], params=params)  # noqa: S610
+    if operator == 'file_type':
+        mime_prefix = _FILE_TYPE_ALIASES.get(value.lower(), value)
+        return queryset.filter(asset_metadata__encodingFormat__istartswith=mime_prefix)
+    raise ValueError(f'unknown asset operator: {operator}')  # pragma: no cover
+
+
+_MODIFIED_ALIAS = '_search_latest_version_modified'
+_PUBLISHED_ALIAS = '_search_latest_published_created'
+
+
+def _apply_date_filter(queryset, operator: str, ts: datetime, annotated: set[str]):
+    """Apply a single parsed date operator to the queryset, annotating as needed.
+
+    ``annotated`` is mutated to track which annotation aliases have been added,
+    so repeated date operators (e.g. modified_before AND modified_after) don't
+    re-annotate the queryset and conflict.
+    """
+    if operator == 'created_before':
+        return queryset.filter(created__lt=ts)
+    if operator == 'created_after':
+        return queryset.filter(created__gte=ts)
+    if operator in {'modified_before', 'modified_after'}:
+        if _MODIFIED_ALIAS not in annotated:
+            queryset = _annotate_latest_version_modified(queryset)
+            annotated.add(_MODIFIED_ALIAS)
+        suffix = '__lt' if operator == 'modified_before' else '__gte'
+        return queryset.filter(**{_MODIFIED_ALIAS + suffix: ts})
+    if operator in {'published_before', 'published_after'}:
+        if _PUBLISHED_ALIAS not in annotated:
+            queryset = _annotate_latest_published_created(queryset)
+            annotated.add(_PUBLISHED_ALIAS)
+        suffix = '__lt' if operator == 'published_before' else '__gte'
+        return queryset.filter(**{_PUBLISHED_ALIAS + suffix: ts})
+    raise ValueError(f'unknown date operator: {operator}')  # pragma: no cover
+
+
+def apply_search_filters(
+    queryset: QuerySet[Dandiset],
+    parsed: ParsedSearch,
+    *,
+    user: User | AnonymousUser,
+) -> QuerySet[Dandiset]:
+    """Apply structured operator filters onto a Dandiset queryset.
+
+    Free text in ``parsed`` is *not* applied here — that stays in the existing
+    full-text filter so the two paths can be tested independently.
+    """
+    if not parsed.operators:
+        return queryset
+
+    # `asset_qs` is built lazily so a query with no asset ops pays nothing.
+    # Note on semantics: chaining `.filter()` AND's the conditions on a SINGLE
+    # AssetSearch row, so e.g. `species:mouse approach:ephys` returns dandisets
+    # with at least one asset that satisfies BOTH (cross-key AND on the same
+    # asset). Repeated keys (`species:mouse species:rat`) likewise require a
+    # single asset to match both substrings — same as GitHub's default.
+    asset_qs = None
+    annotated: set[str] = set()
+
+    for key, raw_value in parsed.operators:
+        value = raw_value.strip()
+        if not value:
+            raise SearchSyntaxError(f'Operator "{key}" requires a value (e.g. {key}:something).')
+
+        if key in _DATE_OPS:
+            try:
+                ts = datetime.strptime(value, '%Y-%m-%d').replace(tzinfo=UTC)
+            except ValueError as exc:
+                raise SearchSyntaxError(
+                    f'Invalid date for "{key}": {value!r}. Use YYYY-MM-DD.'
+                ) from exc
+            queryset = _apply_date_filter(queryset, key, ts, annotated)
+        elif key in _ASSET_OPS:
+            if asset_qs is None:
+                asset_qs = AssetSearch.objects.visible_to(user)
+            asset_qs = _apply_asset_filter(asset_qs, key, value)
+
+    if asset_qs is not None:
+        # NOTE perf: jsonb_path_exists with a runtime-built jsonpath cannot
+        # use the existing per-field GIN indexes; the path-scan operators
+        # (species/approach/technique/standard) currently sequential-scan the
+        # asset_search materialized view. The view is small enough today
+        # (~one row per asset) that this is acceptable, but if it becomes a
+        # hot path the fix is expression GIN indexes on each path or
+        # denormalized text columns + trgm_ops indexes.
+        matching_dandiset_ids = asset_qs.values_list('dandiset_id', flat=True).distinct()
+        queryset = queryset.filter(id__in=matching_dandiset_ids)
+
+    return queryset

dandiapi/api/services/search/parser.py

@@ -0,0 +1,113 @@
+"""Gmail-style search query parser for the Dandiset listing endpoint.
+
+Recognizes ``key:value`` operators interleaved with free-text terms.
+Quoted phrases (``"like this"``) are kept together — they can be used as
+operator values (``technique:"spike sorting"``) or to escape a token
+that looks like an operator (``"foo:bar"`` is treated as free text).
+
+Errors are raised explicitly via :class:`SearchSyntaxError`:
+- unbalanced quotes
+- unknown operator key (with a "did you mean?" suggestion when close)
+
+The view layer translates :class:`SearchSyntaxError` into a 400 response.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from difflib import get_close_matches
+import re
+
+OPERATOR_KEYS: frozenset[str] = frozenset(
+    {
+        'created_before',
+        'created_after',
+        'modified_before',
+        'modified_after',
+        'published_before',
+        'published_after',
+        'species',
+        'approach',
+        'technique',
+        'standard',
+        'file_type',
+    }
+)
+
+# A token in the input is one of:
+#   key:"quoted value"       — operator with quoted value
+#   "quoted text"            — quoted free text (escapes operator-like content)
+#   key:value                — operator (key validated against OPERATOR_KEYS)
+#   bare_token               — free text
+#
+# We deliberately match `key:"value"` and `"value"` *before* the bare-token
+# alternative so quoted segments stay together.
+_TOKEN_RE = re.compile(
+    r'(?P<op_key>[a-z_]+):"(?P<op_qval>[^"]*)"'
+    r'|"(?P<free_quoted>[^"]*)"'
+    r'|(?P<bare>\S+)'
+)
+_BARE_OP_RE = re.compile(r'^([a-z_]+):(.+)$')
+
+
+# Defense-in-depth: cap search-term length so an unauthenticated caller can't
+# force the parser/Postgres to materialize a multi-MB query string. Generous
+# enough that any reasonable interactive query fits.
+_MAX_SEARCH_LENGTH = 1024
+
+
+class SearchSyntaxError(ValueError):
+    """Raised when a search query can't be parsed."""
+
+
+@dataclass
+class ParsedSearch:
+    free_text: list[str] = field(default_factory=list)
+    operators: list[tuple[str, str]] = field(default_factory=list)
+
+
+def _check_balanced_quotes(query: str) -> None:
+    if query.count('"') % 2 != 0:
+        raise SearchSyntaxError(
+            'Unbalanced quote in search query. Remove the stray quote, or wrap '
+            'the intended phrase in matched quotes.'
+        )
+
+
+def _validate_operator_key(key: str) -> None:
+    if key in OPERATOR_KEYS:
+        return
+    suggestions = get_close_matches(key, OPERATOR_KEYS, n=1, cutoff=0.6)
+    hint = f' Did you mean "{suggestions[0]}"?' if suggestions else ''
+    raise SearchSyntaxError(
+        f'Unknown search operator "{key}".{hint} '
+        'Wrap the term in double quotes (e.g. "foo:bar") to search for it as text.'
+    )
+
+
+def parse_search(query: str) -> ParsedSearch:
+    parsed = ParsedSearch()
+    if not query:
+        return parsed
+    if len(query) > _MAX_SEARCH_LENGTH:
+        raise SearchSyntaxError(
+            f'Search query is too long ({len(query)} characters; max {_MAX_SEARCH_LENGTH}).'
+        )
+
+    _check_balanced_quotes(query)
+
+    for match in _TOKEN_RE.finditer(query):
+        if (key := match.group('op_key')) is not None:
+            _validate_operator_key(key)
+            parsed.operators.append((key, match.group('op_qval')))
+        elif (free := match.group('free_quoted')) is not None:
+            parsed.free_text.append(free)
+        else:
+            bare = match.group('bare')
+            if op_match := _BARE_OP_RE.match(bare):
+                key = op_match.group(1)
+                _validate_operator_key(key)
+                parsed.operators.append((key, op_match.group(2)))
+            else:
+                parsed.free_text.append(bare)
+    return parsed