diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index a34e7d561..adc98aae0 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -1749,3 +1749,75 @@ class TopIdentifiersResponseSerializer(serializers.Serializer): project_id = serializers.IntegerField() top_identifiers = UserIdentificationCountSerializer(many=True) + + +class ModelAgreementSerializer(serializers.Serializer): + """Verified / agreement rates over the filtered Occurrence set. + + `agreed_exact_count` is a subset of `agreed_any_rank_count` by + construction — an exact match implies the LCA is the taxon itself. + `*_pct` percentages are 0.0..1.0 (not 0..100). + + Denominator note: `agreed_*_pct` divide by `verified_with_prediction_count` + (verified occurrences that *also* have a machine prediction), NOT by + `verified_count`. A verified occurrence with no machine prediction can't + agree or disagree — including it in the denominator would drag the rate + down without representing actual model disagreement. `no_prediction_count` + is surfaced so the consumer can see how many such occurrences exist. + + Optional rank threshold: when the caller passes + `?agreement_coarsest_rank=FAMILY`, the response also includes + `agreed_coarser_rank_*` counting only LCAs at that rank or deeper. The + threshold rank is echoed in `agreement_coarsest_rank`. When the param is + absent, the coarser-rank fields are null and `agreement_coarsest_rank` + is null. + """ + + project_id = serializers.IntegerField() + total_occurrences = serializers.IntegerField() + verified_count = serializers.IntegerField(help_text="Occurrences with at least one non-withdrawn identification.") + verified_pct = serializers.FloatField( + min_value=0.0, + max_value=1.0, + help_text="verified_count / total_occurrences", + ) + verified_with_prediction_count = serializers.IntegerField( + help_text="Verified occurrences that also have a machine prediction (denominator for agreed_*_pct)." + ) + no_prediction_count = serializers.IntegerField( + help_text="Verified occurrences with no machine prediction (excluded from agreement denominator)." + ) + agreed_exact_count = serializers.IntegerField() + agreed_exact_pct = serializers.FloatField( + min_value=0.0, + max_value=1.0, + help_text="agreed_exact_count / verified_with_prediction_count", + ) + agreed_any_rank_count = serializers.IntegerField( + help_text="Exact matches plus disagreements whose LCA is at any real rank (UNKNOWN excluded)." + ) + agreed_any_rank_pct = serializers.FloatField( + min_value=0.0, + max_value=1.0, + help_text="agreed_any_rank_count / verified_with_prediction_count", + ) + agreement_coarsest_rank = serializers.CharField( + allow_null=True, + required=False, + help_text="Threshold rank from ?agreement_coarsest_rank query param. Null when the param is absent.", + ) + agreed_coarser_rank_count = serializers.IntegerField( + allow_null=True, + required=False, + help_text=( + "Exact matches plus disagreements whose LCA is at `agreement_coarsest_rank` or deeper. " + "Null when no threshold was supplied." + ), + ) + agreed_coarser_rank_pct = serializers.FloatField( + min_value=0.0, + max_value=1.0, + allow_null=True, + required=False, + help_text="agreed_coarser_rank_count / verified_with_prediction_count. Null when no threshold supplied.", + ) diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 5a6d5aece..d0549cb83 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -32,7 +32,7 @@ from ami.base.views import ProjectMixin from ami.main.api.schemas import limit_doc_param, project_id_doc_param from ami.main.api.serializers import TagSerializer -from ami.main.models_future.occurrence import top_identifiers_for_project +from ami.main.models_future.occurrence import model_agreement_for_project, top_identifiers_for_project from ami.utils.requests import get_default_classification_threshold from ami.utils.storages import ConnectionTestResult @@ -56,6 +56,7 @@ Tag, TaxaList, Taxon, + TaxonRank, User, update_detection_counts, ) @@ -72,6 +73,7 @@ EventSerializer, EventTimelineSerializer, IdentificationSerializer, + ModelAgreementSerializer, OccurrenceListSerializer, OccurrenceSerializer, PageListSerializer, @@ -1168,6 +1170,24 @@ def filter_queryset(self, request, queryset, view): return queryset +OCCURRENCE_FILTER_BACKENDS = ( + CustomOccurrenceDeterminationFilter, + OccurrenceCollectionFilter, + OccurrenceAlgorithmFilter, + OccurrenceDateFilter, + OccurrenceVerified, + OccurrenceVerifiedByMeFilter, + OccurrenceTaxaListFilter, +) + +OCCURRENCE_FILTERSET_FIELDS = ( + "event", + "deployment", + "determination__rank", + "detections__source_image", +) + + class OccurrenceViewSet(DefaultViewSet, ProjectMixin): """ API endpoint that allows occurrences to be viewed or edited. @@ -1177,22 +1197,8 @@ class OccurrenceViewSet(DefaultViewSet, ProjectMixin): queryset = Occurrence.objects.all() serializer_class = OccurrenceSerializer - # filter_backends = [CustomDeterminationFilter, DjangoFilterBackend, NullsLastOrderingFilter, SearchFilter] - filter_backends = DefaultViewSetMixin.filter_backends + [ - CustomOccurrenceDeterminationFilter, - OccurrenceCollectionFilter, - OccurrenceAlgorithmFilter, - OccurrenceDateFilter, - OccurrenceVerified, - OccurrenceVerifiedByMeFilter, - OccurrenceTaxaListFilter, - ] - filterset_fields = [ - "event", - "deployment", - "determination__rank", - "detections__source_image", - ] + filter_backends = DefaultViewSetMixin.filter_backends + list(OCCURRENCE_FILTER_BACKENDS) + filterset_fields = list(OCCURRENCE_FILTERSET_FIELDS) ordering_fields = [ "created_at", "updated_at", @@ -1290,6 +1296,11 @@ class OccurrenceStatsViewSet(viewsets.GenericViewSet, ProjectMixin): permission_classes = [IsActiveStaffOrReadOnly] require_project = True + # Filter machinery for actions that opt into `self.filter_queryset(...)`. + # `top_identifiers` doesn't call it, so its behavior is unchanged. + queryset = Occurrence.objects.none() + filter_backends = [DjangoFilterBackend, *OCCURRENCE_FILTER_BACKENDS] + filterset_fields = list(OCCURRENCE_FILTERSET_FIELDS) @extend_schema( parameters=[project_id_doc_param, limit_doc_param], @@ -1320,6 +1331,48 @@ def top_identifiers(self, request): ) return Response(serializer.data) + @extend_schema( + parameters=[project_id_doc_param], + responses=ModelAgreementSerializer, + ) + @action(detail=False, methods=["get"], url_path="model-agreement") + def model_agreement(self, request): + """Verified / human↔model agreement rates over the filtered occurrence set. + + Accepts every query param the `/occurrences/` list endpoint accepts. + Reuses `apply_default_filters` so `apply_defaults=false` bypasses + project default taxa lists + score thresholds. + + Optional ?agreement_coarsest_rank= adds `agreed_coarser_rank_*` + counts — LCAs at the given rank or deeper. Valid values: any + TaxonRank name (FAMILY, GENUS, etc.); invalid → 400. + """ + project = self.get_active_project() + assert project is not None # require_project=True guarantees this + if not Project.objects.visible_for_user(request.user).filter(pk=project.pk).exists(): + raise NotFound("Project not found.") + + coarsest_rank_param = request.query_params.get("agreement_coarsest_rank") + coarsest_rank = None + if coarsest_rank_param: + try: + coarsest_rank = TaxonRank[coarsest_rank_param.upper()] + except KeyError: + valid = ", ".join(r.name for r in TaxonRank if r.name != "UNKNOWN") + raise api_exceptions.ValidationError( + {"agreement_coarsest_rank": f"Invalid rank '{coarsest_rank_param}'. Must be one of: {valid}."} + ) + if coarsest_rank == TaxonRank.UNKNOWN: + raise api_exceptions.ValidationError( + {"agreement_coarsest_rank": "UNKNOWN is not a valid threshold rank."} + ) + + base_qs = Occurrence.objects.filter(project=project).valid().apply_default_filters(project, request) + filtered_qs = self.filter_queryset(base_qs) + payload = model_agreement_for_project(filtered_qs, coarsest_rank=coarsest_rank) + payload["project_id"] = project.pk + return Response(ModelAgreementSerializer(payload, context={"request": request}).data) + class TaxonTaxaListFilter(filters.BaseFilterBackend): """ diff --git a/ami/main/models_future/occurrence.py b/ami/main/models_future/occurrence.py index 6f599cbe0..cdce45cb8 100644 --- a/ami/main/models_future/occurrence.py +++ b/ami/main/models_future/occurrence.py @@ -12,13 +12,43 @@ from typing import TYPE_CHECKING -from django.db.models import Count, Prefetch, Q, QuerySet +from django.db.models import Count, OuterRef, Prefetch, Q, QuerySet, Subquery -from ami.main.models import Project, User +from ami.main.models import Project, TaxonRank, User if TYPE_CHECKING: from ami.main.models import Classification, Identification, Occurrence +TaxonTuple = tuple[int, str, list[dict]] + + +def lca_rank_between(a: TaxonTuple, b: TaxonTuple) -> TaxonRank | None: + """Most-specific shared ancestor rank between two taxa. + + Inputs are ``(taxon_id, rank_str, parents_json)`` triples where + ``parents_json`` is ordered root → immediate parent (Taxon.parents_json layout). + + The taxon itself counts as part of its own ancestor chain — passing the + same taxon twice returns that taxon's rank. Returns ``None`` when the two + chains share no ancestor at a real taxonomic rank. + + ``TaxonRank.UNKNOWN`` is excluded from the candidate set even though it + sorts after SPECIES in OrderedEnum definition order — it isn't a real + taxonomic rank and treating it as deeper-than-ORDER produces false + under-order agreements when an UNKNOWN ancestor happens to be shared. + """ + chain_a = [(p["id"], TaxonRank(p["rank"])) for p in a[2]] + [(a[0], TaxonRank(a[1]))] + chain_b_ids = {p["id"] for p in b[2]} | {b[0]} + + deepest: TaxonRank | None = None + for tid, rank in chain_a: + if rank == TaxonRank.UNKNOWN: + continue + if tid in chain_b_ids: + if deepest is None or rank > deepest: + deepest = rank + return deepest + def _detections_prefetch(*, ordering: tuple[str, ...], with_source_image: bool) -> Prefetch: from ami.main.models import Classification, Detection @@ -133,6 +163,140 @@ def detection_image_urls_from_prefetch(occurrence: Occurrence, limit: int | None return [get_media_url(det.path) for det in detections] +def model_agreement_for_project( + queryset: QuerySet[Occurrence], + coarsest_rank: TaxonRank | None = None, +) -> dict: + """Verified / agreement stats over a pre-filtered Occurrence queryset. + + The queryset MUST already be filtered to the project + user-supplied + filters (caller wires apply_default_filters + OccurrenceFilter). This + function adds the annotations it needs and returns a dict matching + ModelAgreementSerializer's field set (without project_id — the view + layer adds that). + + "Verified" means the occurrence has at least one non-withdrawn + Identification. "Model prediction" means the Classification chosen by + BEST_MACHINE_PREDICTION_ORDER. "Any-rank" agreement means the user's + taxon and the model's prediction share an ancestor at any real rank + (UNKNOWN excluded) — exact matches included. The upstream filter (e.g. + a Lepidoptera include list) is what bounds the meaningful scope, not + a hardcoded rank threshold in this function. + + When ``coarsest_rank`` is supplied, additionally compute "coarser-rank" + agreement: the LCA must be at ``coarsest_rank`` or deeper (e.g. passing + FAMILY only counts LCAs at FAMILY, GENUS, or SPECIES). Exact matches + always count regardless of rank. + + Performance: the heavy work — correlated subqueries over Identification + and Classification — is scoped to the verified set, which is typically + a tiny fraction of total occurrences. Computing those subqueries over + the full filtered queryset would do 99% wasted work picking the "best + user identification" for occurrences that have none. + + Step 1: total_occurrences = SQL Count(*). + Step 2: Fetch the verified set with (pk, best_user_taxon_id, + best_machine_prediction_taxon_id). Both correlated subqueries + evaluate only on verified rows. + Step 3: Bucket counts in Python (set is small). + Step 4: Dedupe disagreement to distinct (user, machine) pairs and run + one LCA per pair. + + Bench against project 18 (43,149 occurrences, 45 verified): ~80ms cold. + """ + import collections + + from ami.main.models import BEST_IDENTIFICATION_ORDER, Identification, Taxon + + # Default filters can join Identification (verified_by_me) and Taxon + # parents_json (taxa_list_id) which inflates row count if not deduped. + # Dedupe up front so total + verified counts share one canonical set. + queryset = queryset.distinct() + total = queryset.count() + + best_user_ident = Identification.objects.filter(occurrence=OuterRef("pk"), withdrawn=False).order_by( + *BEST_IDENTIFICATION_ORDER + ) + + verified_rows = list( + queryset.filter(identifications__withdrawn=False) + .distinct() + .with_best_machine_prediction() # type: ignore[attr-defined] + .annotate(best_user_taxon_id=Subquery(best_user_ident.values("taxon_id")[:1])) + .values("pk", "best_machine_prediction_taxon_id", "best_user_taxon_id") + ) + + verified = len(verified_rows) + no_prediction = sum(1 for r in verified_rows if r["best_machine_prediction_taxon_id"] is None) + verified_with_pred = verified - no_prediction + agreed_exact = sum( + 1 + for r in verified_rows + if r["best_machine_prediction_taxon_id"] is not None + and r["best_user_taxon_id"] == r["best_machine_prediction_taxon_id"] + ) + + # Dedupe disagreement pairs so each (user_taxon, machine_taxon) LCA runs once. + pair_counts: collections.Counter = collections.Counter() + for r in verified_rows: + m_id = r["best_machine_prediction_taxon_id"] + u_id = r["best_user_taxon_id"] + if m_id is None or u_id is None or u_id == m_id: + continue + pair_counts[(u_id, m_id)] += 1 + + needed_taxa_ids: set[int] = set() + for u_id, m_id in pair_counts: + needed_taxa_ids.add(u_id) + needed_taxa_ids.add(m_id) + + taxa_by_id: dict[int, TaxonTuple] = {} + if needed_taxa_ids: + for t in Taxon.objects.filter(pk__in=needed_taxa_ids): + parents = [ + {"id": p.id, "rank": p.rank.name if hasattr(p.rank, "name") else p.rank} for p in t.parents_json + ] + taxa_by_id[t.pk] = (t.pk, t.rank, parents) + + any_rank_disagreement_count = 0 + coarser_rank_disagreement_count = 0 + for (u_id, m_id), count in pair_counts.items(): + u = taxa_by_id.get(u_id) + m = taxa_by_id.get(m_id) + if not u or not m: + continue + lca = lca_rank_between(u, m) + if lca is None: + continue + any_rank_disagreement_count += count + if coarsest_rank is not None and lca >= coarsest_rank: + coarser_rank_disagreement_count += count + + agreed_any_rank = agreed_exact + any_rank_disagreement_count + agreed_coarser_rank = agreed_exact + coarser_rank_disagreement_count + + def _pct(num: int, denom: int) -> float: + return round(num / denom, 4) if denom else 0.0 + + payload: dict = { + "total_occurrences": total, + "verified_count": verified, + "verified_pct": _pct(verified, total), + "verified_with_prediction_count": verified_with_pred, + "no_prediction_count": no_prediction, + "agreed_exact_count": agreed_exact, + "agreed_exact_pct": _pct(agreed_exact, verified_with_pred), + "agreed_any_rank_count": agreed_any_rank, + "agreed_any_rank_pct": _pct(agreed_any_rank, verified_with_pred), + "agreement_coarsest_rank": coarsest_rank.name if coarsest_rank is not None else None, + "agreed_coarser_rank_count": agreed_coarser_rank if coarsest_rank is not None else None, + "agreed_coarser_rank_pct": ( + _pct(agreed_coarser_rank, verified_with_pred) if coarsest_rank is not None else None + ), + } + return payload + + def top_identifiers_for_project(project: Project) -> QuerySet[User]: """Project users ranked by distinct occurrences they identified. diff --git a/ami/main/tests.py b/ami/main/tests.py index 517c4c87b..1ae5c54f9 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -4679,6 +4679,222 @@ def test_source_image_cached_counts_refresh_on_threshold_change(self): ) +class TestLcaRankBetween(TestCase): + """Pure-Python LCA over (taxon_id, rank, parents_json) tuples. + + Inputs encode each taxon as ``(id, rank_str, [{"id": int, "rank": str}, ...])`` + where the parents list is ordered root → immediate-parent (matches + Taxon.parents_json layout). + """ + + GENUS_NOCTUA = ( + 101, + "GENUS", + [ + {"id": 1, "rank": "KINGDOM"}, + {"id": 4, "rank": "ORDER"}, + {"id": 30, "rank": "FAMILY"}, + ], + ) + SPECIES_NOCTUA_PRONUBA = ( + 201, + "SPECIES", + [ + {"id": 1, "rank": "KINGDOM"}, + {"id": 4, "rank": "ORDER"}, + {"id": 30, "rank": "FAMILY"}, + {"id": 101, "rank": "GENUS"}, + ], + ) + SPECIES_NOCTUA_FIMBRIATA = ( + 202, + "SPECIES", + [ + {"id": 1, "rank": "KINGDOM"}, + {"id": 4, "rank": "ORDER"}, + {"id": 30, "rank": "FAMILY"}, + {"id": 101, "rank": "GENUS"}, + ], + ) + SPECIES_DIFFERENT_FAMILY = ( + 301, + "SPECIES", + [ + {"id": 1, "rank": "KINGDOM"}, + {"id": 4, "rank": "ORDER"}, + {"id": 99, "rank": "FAMILY"}, + ], + ) + SPECIES_DIFFERENT_ORDER = ( + 401, + "SPECIES", + [ + {"id": 1, "rank": "KINGDOM"}, + {"id": 5, "rank": "ORDER"}, + ], + ) + + def test_identical_taxa_lca_is_self_rank(self): + from ami.main.models_future.occurrence import lca_rank_between + + rank = lca_rank_between(self.SPECIES_NOCTUA_PRONUBA, self.SPECIES_NOCTUA_PRONUBA) + self.assertEqual(rank, TaxonRank.SPECIES) + + def test_sister_species_share_genus(self): + from ami.main.models_future.occurrence import lca_rank_between + + rank = lca_rank_between(self.SPECIES_NOCTUA_PRONUBA, self.SPECIES_NOCTUA_FIMBRIATA) + self.assertEqual(rank, TaxonRank.GENUS) + + def test_genus_vs_species_in_same_genus(self): + from ami.main.models_future.occurrence import lca_rank_between + + rank = lca_rank_between(self.GENUS_NOCTUA, self.SPECIES_NOCTUA_PRONUBA) + self.assertEqual(rank, TaxonRank.GENUS) + + def test_different_family_same_order(self): + from ami.main.models_future.occurrence import lca_rank_between + + rank = lca_rank_between(self.SPECIES_NOCTUA_PRONUBA, self.SPECIES_DIFFERENT_FAMILY) + self.assertEqual(rank, TaxonRank.ORDER) + + def test_different_order_same_kingdom(self): + from ami.main.models_future.occurrence import lca_rank_between + + rank = lca_rank_between(self.SPECIES_NOCTUA_PRONUBA, self.SPECIES_DIFFERENT_ORDER) + self.assertEqual(rank, TaxonRank.KINGDOM) + + def test_no_shared_ancestor_returns_none(self): + from ami.main.models_future.occurrence import lca_rank_between + + rootless = (501, "SPECIES", []) + rank = lca_rank_between(rootless, self.SPECIES_NOCTUA_PRONUBA) + self.assertIsNone(rank) + + def test_unknown_rank_excluded_from_lca(self): + """TaxonRank.UNKNOWN sorts after SPECIES in OrderedEnum definition order, + so without explicit exclusion `UNKNOWN >= ORDER` would be True and a + shared UNKNOWN ancestor would wrongly count as under-order agreement. + """ + from ami.main.models_future.occurrence import lca_rank_between + + # Both chains share a KINGDOM ancestor and an UNKNOWN ancestor; the LCA + # at a real taxonomic rank is KINGDOM, not UNKNOWN. + unknown_a = ( + 701, + "SPECIES", + [ + {"id": 1, "rank": "KINGDOM"}, + {"id": 999, "rank": "UNKNOWN"}, + ], + ) + unknown_b = ( + 702, + "SPECIES", + [ + {"id": 1, "rank": "KINGDOM"}, + {"id": 999, "rank": "UNKNOWN"}, + ], + ) + rank = lca_rank_between(unknown_a, unknown_b) + self.assertEqual(rank, TaxonRank.KINGDOM) + + +class TestModelAgreementForProject(APITestCase): + """Aggregation function over a filtered Occurrence queryset. + + Covers four bucket transitions: unverified, verified+exact-agreed, + verified+any-rank-agreed (no threshold), verified+disagreed-no-shared-rank. + Optional coarsest_rank threshold cases handled in the viewset tests below. + """ + + def setUp(self) -> None: + project, deployment = setup_test_project() + create_taxa(project=project) + create_captures(deployment=deployment) + # Add a sibling family + species under the same ORDER so we can exercise + # the "different family, same order → under-order" bucket. + lepidoptera = Taxon.objects.get(name="Lepidoptera", projects=project) + pieridae = Taxon.objects.create(name="Pieridae", parent=lepidoptera, rank=TaxonRank.FAMILY.name) + pieridae.projects.add(project) + pieris = Taxon.objects.create(name="Pieris brassicae", parent=pieridae, rank=TaxonRank.SPECIES.name) + pieris.projects.add(project) + # Use Vanessa atalanta as the baseline machine prediction so all + # occurrences start with a known classification. + self.vanessa_atalanta = Taxon.objects.get(name="Vanessa atalanta", projects=project) + create_occurrences(deployment=deployment, num=4, taxon=self.vanessa_atalanta) + # Populate parents_json on every taxon — fixtures don't do this. + Taxon.objects.update_all_parents() + self.project = project + self.deployment = deployment + self.vanessa_cardui = Taxon.objects.get(name="Vanessa cardui", projects=project) + self.pieris_brassicae = pieris + self.user = User.objects.create_user(email="ider@insectai.org") # type: ignore + + def _identify(self, occurrence: Occurrence, taxon: Taxon) -> Identification: + return Identification.objects.create(user=self.user, occurrence=occurrence, taxon=taxon) + + def test_empty_project_returns_zeros_not_nans(self): + from ami.main.models_future.occurrence import model_agreement_for_project + + empty_project = Project.objects.create(name="empty") + result = model_agreement_for_project(Occurrence.objects.filter(project=empty_project)) + self.assertEqual(result["total_occurrences"], 0) + self.assertEqual(result["verified_count"], 0) + self.assertEqual(result["verified_pct"], 0.0) + self.assertEqual(result["agreed_exact_pct"], 0.0) + self.assertEqual(result["agreed_any_rank_pct"], 0.0) + # No threshold passed → coarser-rank fields null. + self.assertIsNone(result["agreement_coarsest_rank"]) + self.assertIsNone(result["agreed_coarser_rank_count"]) + self.assertIsNone(result["agreed_coarser_rank_pct"]) + + def test_buckets_canonical_cases(self): + from ami.main.models_future.occurrence import model_agreement_for_project + + occurrences = list(Occurrence.objects.filter(project=self.project).order_by("pk")) + self.assertEqual(len(occurrences), 4) + # 0: verified, machine == user (exact agreement at SPECIES) + self._identify(occurrences[0], self.vanessa_atalanta) + # 1: verified, sister species (LCA at GENUS) + self._identify(occurrences[1], self.vanessa_cardui) + # 2: verified, different family same order (LCA at ORDER) + self._identify(occurrences[2], self.pieris_brassicae) + # 3: unverified + + result = model_agreement_for_project(Occurrence.objects.filter(project=self.project)) + self.assertEqual(result["total_occurrences"], 4) + self.assertEqual(result["verified_count"], 3) + self.assertEqual(result["agreed_exact_count"], 1) + self.assertEqual(result["agreed_any_rank_count"], 3) + self.assertAlmostEqual(result["verified_pct"], 0.75) + self.assertAlmostEqual(result["agreed_exact_pct"], 1 / 3, places=3) + self.assertAlmostEqual(result["agreed_any_rank_pct"], 1.0) + + def test_coarsest_rank_threshold_filters_shallow_lcas(self): + """With coarsest_rank=FAMILY, an ORDER-only LCA pair is excluded.""" + from ami.main.models import TaxonRank + from ami.main.models_future.occurrence import model_agreement_for_project + + occurrences = list(Occurrence.objects.filter(project=self.project).order_by("pk")) + # 0: exact (SPECIES) — counts in both + self._identify(occurrences[0], self.vanessa_atalanta) + # 1: sister species (LCA = GENUS, deeper than FAMILY) — counts in both + self._identify(occurrences[1], self.vanessa_cardui) + # 2: different family same order (LCA = ORDER, NOT >= FAMILY) — counts in any_rank only + self._identify(occurrences[2], self.pieris_brassicae) + + result = model_agreement_for_project( + Occurrence.objects.filter(project=self.project), + coarsest_rank=TaxonRank.FAMILY, + ) + self.assertEqual(result["agreed_any_rank_count"], 3) + self.assertEqual(result["agreement_coarsest_rank"], "FAMILY") + # exact + GENUS LCA = 2; ORDER LCA excluded + self.assertEqual(result["agreed_coarser_rank_count"], 2) + self.assertAlmostEqual(result["agreed_coarser_rank_pct"], 2 / 3, places=3) + + class TestOccurrenceStatsViewSet(APITestCase): """Covers /api/v2/occurrences/stats/top-identifiers/. @@ -4761,3 +4977,129 @@ def test_registration_order_preserves_occurrence_retrieve(self): retrieve_response = self.client.get(f"/api/v2/occurrences/{occurrence.pk}/?project_id={self.project.pk}") self.assertEqual(stats_response.status_code, 200, "stats URL must resolve") self.assertEqual(retrieve_response.status_code, 200, "occurrence retrieve must still work") + + # ----- /occurrences/stats/model-agreement/ ----- + + agreement_url = "/api/v2/occurrences/stats/model-agreement/" + + def test_agreement_no_project_id_returns_400(self): + response = self.client.get(self.agreement_url) + self.assertEqual(response.status_code, 400) + + def test_agreement_draft_project_404_for_anon(self): + self.project.draft = True + self.project.save() + response = self.client.get(f"{self.agreement_url}?project_id={self.project.pk}") + self.assertEqual(response.status_code, 404) + + def test_agreement_empty_returns_zero_pcts(self): + response = self.client.get(f"{self.agreement_url}?project_id={self.project.pk}") + self.assertEqual(response.status_code, 200) + body = response.json() + self.assertEqual(body["project_id"], self.project.pk) + self.assertEqual(body["total_occurrences"], 4) + self.assertEqual(body["verified_count"], 0) + self.assertEqual(body["verified_pct"], 0.0) + self.assertEqual(body["agreed_exact_pct"], 0.0) + self.assertEqual(body["agreed_any_rank_pct"], 0.0) + # No ?agreement_coarsest_rank → threshold + coarser fields null. + self.assertIsNone(body["agreement_coarsest_rank"]) + self.assertIsNone(body["agreed_coarser_rank_count"]) + self.assertIsNone(body["agreed_coarser_rank_pct"]) + + def test_agreement_happy_path(self): + """One verified occurrence; user agrees with the machine prediction → exact match. + + The fixture creates a single classification per occurrence via + `create_occurrences()`, which uses a random taxon. We identify the + first occurrence with that same taxon to force an exact match. + """ + occurrence = Occurrence.objects.filter(project=self.project).order_by("pk").first() + # The machine prediction is whatever `create_occurrences()` picked — match it. + machine_taxon = occurrence.detections.first().classifications.first().taxon + Taxon.objects.update_all_parents() + Identification.objects.create(user=self.alice, occurrence=occurrence, taxon=machine_taxon) + + response = self.client.get(f"{self.agreement_url}?project_id={self.project.pk}") + self.assertEqual(response.status_code, 200) + body = response.json() + self.assertEqual(body["total_occurrences"], 4) + self.assertEqual(body["verified_count"], 1) + self.assertEqual(body["verified_with_prediction_count"], 1) + self.assertEqual(body["no_prediction_count"], 0) + self.assertEqual(body["agreed_exact_count"], 1) + self.assertEqual(body["agreed_any_rank_count"], 1) + + def test_agreement_any_rank_bucket(self): + """Disagreement at species but same genus → counted as any-rank agreement, not exact. + + Pick the machine prediction's sister species (same parent genus) for the + identification. LCA between the two species is GENUS, so the occurrence + falls into the any-rank bucket without contributing to agreed_exact_count. + """ + occurrence = Occurrence.objects.filter(project=self.project).order_by("pk").first() + machine_taxon = occurrence.detections.first().classifications.first().taxon + # Sister species: same parent (genus Vanessa), different SPECIES. + sister = ( + Taxon.objects.filter(parent=machine_taxon.parent, rank=TaxonRank.SPECIES.name) + .exclude(pk=machine_taxon.pk) + .first() + ) + self.assertIsNotNone(sister, "Test fixture must have a sister species under the same genus") + Taxon.objects.update_all_parents() + Identification.objects.create(user=self.alice, occurrence=occurrence, taxon=sister) + + response = self.client.get(f"{self.agreement_url}?project_id={self.project.pk}") + self.assertEqual(response.status_code, 200) + body = response.json() + self.assertEqual(body["verified_count"], 1) + self.assertEqual(body["verified_with_prediction_count"], 1) + self.assertEqual(body["agreed_exact_count"], 0) + self.assertEqual(body["agreed_any_rank_count"], 1) + # 0/1 exact, 1/1 any-rank + self.assertEqual(body["agreed_exact_pct"], 0.0) + self.assertEqual(body["agreed_any_rank_pct"], 1.0) + + def test_agreement_coarsest_rank_invalid_returns_400(self): + response = self.client.get( + f"{self.agreement_url}?project_id={self.project.pk}&agreement_coarsest_rank=GARBAGE" + ) + self.assertEqual(response.status_code, 400) + self.assertIn("agreement_coarsest_rank", response.json()) + + def test_agreement_coarsest_rank_unknown_rejected(self): + """UNKNOWN is a real enum member but not a meaningful threshold.""" + response = self.client.get( + f"{self.agreement_url}?project_id={self.project.pk}&agreement_coarsest_rank=UNKNOWN" + ) + self.assertEqual(response.status_code, 400) + + def test_agreement_coarsest_rank_echoed_in_response(self): + response = self.client.get(f"{self.agreement_url}?project_id={self.project.pk}&agreement_coarsest_rank=family") + self.assertEqual(response.status_code, 200) + body = response.json() + # Param is case-insensitive; response echoes enum name (uppercase). + self.assertEqual(body["agreement_coarsest_rank"], "FAMILY") + # No verified occurrences in this fixture → coarser fields present but zero. + self.assertEqual(body["agreed_coarser_rank_count"], 0) + self.assertEqual(body["agreed_coarser_rank_pct"], 0.0) + + def test_agreement_filter_passthrough(self): + """`?deployment=` should narrow the set.""" + other_deployment = Deployment.objects.create(name="other", project=self.project) + response = self.client.get( + f"{self.agreement_url}?project_id={self.project.pk}&deployment={other_deployment.pk}" + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["total_occurrences"], 0) + + def test_agreement_apply_defaults_false_bypasses_score_threshold(self): + """A score threshold filters out occurrences; apply_defaults=false restores them.""" + self.project.default_filters_score_threshold = 0.99 + self.project.save() + gated = self.client.get(f"{self.agreement_url}?project_id={self.project.pk}").json() + bypassed = self.client.get(f"{self.agreement_url}?project_id={self.project.pk}&apply_defaults=false").json() + self.assertGreaterEqual(bypassed["total_occurrences"], gated["total_occurrences"]) + # Sanity: with threshold=0.99 and fixture's score=0.9, gated should be 0. + self.assertEqual(gated["total_occurrences"], 0) + self.assertEqual(bypassed["total_occurrences"], 4) diff --git a/docs/claude/reference/api-stats-pattern.md b/docs/claude/reference/api-stats-pattern.md index 0a2cd20d2..f802d6ad8 100644 --- a/docs/claude/reference/api-stats-pattern.md +++ b/docs/claude/reference/api-stats-pattern.md @@ -232,7 +232,7 @@ into pagination only if the kind genuinely needs it): - `GET /occurrences/stats/top-identifiers/` — done (this PR) - `GET /occurrences/stats/identifications-summary/` — total / distinct / verified counts -- `GET /occurrences/stats/human-model-agreement/` — model agreement rate +- `GET /occurrences/stats/model-agreement/` — model agreement rate - `GET /occurrences/stats/identifications-by-species/` — per-taxon ID counts - `GET /occurrences/stats/timeline/` — Plotly-shaped time series - `GET /deployments/stats/processed-images/` — processed images per station diff --git a/ui/src/data-services/hooks/occurrences/stats/useModelAgreement.ts b/ui/src/data-services/hooks/occurrences/stats/useModelAgreement.ts new file mode 100644 index 000000000..d783103df --- /dev/null +++ b/ui/src/data-services/hooks/occurrences/stats/useModelAgreement.ts @@ -0,0 +1,71 @@ +import { API_ROUTES, API_URL } from 'data-services/constants' +import { useAuthorizedQuery } from '../../auth/useAuthorizedQuery' + +interface ModelAgreementResponse { + project_id: number + total_occurrences: number + verified_count: number + verified_pct: number + verified_with_prediction_count: number + no_prediction_count: number + agreed_exact_count: number + agreed_exact_pct: number + agreed_any_rank_count: number + agreed_any_rank_pct: number + // Only populated when the caller passes ?agreement_coarsest_rank=. + agreement_coarsest_rank: string | null + agreed_coarser_rank_count: number | null + agreed_coarser_rank_pct: number | null +} + +type FilterPrimitive = string | number | boolean +type FilterValue = FilterPrimitive | FilterPrimitive[] | null | undefined + +// Accepts an arbitrary filter map so the occurrence list page's filter state +// can be threaded through unchanged (deployment, event, taxon, score +// thresholds, apply_defaults, etc). Arrays are appended as repeated query +// params so multi-select filters (e.g. `algorithm`, `not_algorithm`, which +// the backend reads via `request.query_params.getlist(...)`) survive. +export const useModelAgreement = ( + projectId?: string, + filters?: Record +) => { + const url = `${API_URL}/${API_ROUTES.OCCURRENCES}/stats/model-agreement/` + + const params = new URLSearchParams() + if (projectId) params.set('project_id', projectId) + if (filters) { + Object.entries(filters).forEach(([key, value]) => { + if (value === undefined || value === null || value === '') return + if (Array.isArray(value)) { + value.forEach((item) => { + if (item !== undefined && item !== null && item !== '') { + params.append(key, String(item)) + } + }) + return + } + params.set(key, String(value)) + }) + } + const queryString = params.toString() + + const { data, isLoading, isFetching, error } = + useAuthorizedQuery({ + queryKey: [ + API_ROUTES.OCCURRENCES, + 'stats', + 'model-agreement', + projectId, + queryString, + ], + url: `${url}?${queryString}`, + }) + + return { + data, + isLoading, + isFetching, + error, + } +}