From 8688e7f0740e7bed8d9594d86b6a51d7673355f3 Mon Sep 17 00:00:00 2001 From: Anna Viklund Date: Thu, 14 May 2026 11:23:13 +0200 Subject: [PATCH 01/13] layout: use compact gallery for projects --- ui/src/pages/projects/project-gallery.tsx | 2 -- 1 file changed, 2 deletions(-) diff --git a/ui/src/pages/projects/project-gallery.tsx b/ui/src/pages/projects/project-gallery.tsx index 80d76d139..d15b91f5a 100644 --- a/ui/src/pages/projects/project-gallery.tsx +++ b/ui/src/pages/projects/project-gallery.tsx @@ -34,7 +34,6 @@ export const ProjectGallery = ({ return ( ( @@ -49,7 +48,6 @@ export const ProjectGallery = ({ /> )} - style={{ gridTemplateColumns: '1fr 1fr 1fr' }} /> ) } From 10292ce58a43a31be14daaccf4f6a770262a43d2 Mon Sep 17 00:00:00 2001 From: Anna Viklund Date: Thu, 14 May 2026 11:23:34 +0200 Subject: [PATCH 02/13] chore: bump page size for projects to 40 --- ui/src/pages/projects/projects.tsx | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ui/src/pages/projects/projects.tsx b/ui/src/pages/projects/projects.tsx index 267ec93f0..323181bb3 100644 --- a/ui/src/pages/projects/projects.tsx +++ b/ui/src/pages/projects/projects.tsx @@ -12,7 +12,6 @@ import { UserPermission } from 'utils/user/types' import { useUser } from 'utils/user/userContext' import { useUserInfo } from 'utils/user/userInfoContext' import { useSelectedView } from 'utils/useSelectedView' -import { useWindowSize } from 'utils/useWindowSize' import { ProjectGallery } from './project-gallery' export const TABS = { @@ -25,10 +24,7 @@ export const Projects = () => { const { userInfo } = useUserInfo() const { selectedView: selectedTab, setSelectedView: setSelectedTab } = useSelectedView(user.loggedIn ? TABS.MY_PROJECTS : TABS.ALL_PROJECTS) - const [windowWidth] = useWindowSize() - const { pagination, setPage } = usePagination({ - perPage: windowWidth > 1024 ? 21 : 20, // Adjust page size based on page width to avoid gallery gaps - }) + const { pagination, setPage } = usePagination({ perPage: 40 }) const filters = user.loggedIn && selectedTab === TABS.MY_PROJECTS ? [{ field: 'user_id', value: userInfo?.id }] From 8e449b77157f544800f8fb33837f06920e0943ab Mon Sep 17 00:00:00 2001 From: Anna Viklund Date: Thu, 14 May 2026 15:49:52 +0200 Subject: [PATCH 03/13] feat: make it possible to see and change sort order from sort control --- ui/package.json | 2 +- .../design-system/components/sort-control.tsx | 98 +++++++++++++------ ui/src/utils/language.ts | 2 + ui/yarn.lock | 10 +- 4 files changed, 77 insertions(+), 35 deletions(-) diff --git a/ui/package.json b/ui/package.json index 7c32af6e1..bd8f41ac5 100644 --- a/ui/package.json +++ b/ui/package.json @@ -29,7 +29,7 @@ "leaflet": "^1.9.3", "lodash": "^4.17.21", "lucide-react": "^1.0.1", - "nova-ui-kit": "^1.1.34", + "nova-ui-kit": "^1.1.36", "plotly.js": "^2.25.2", "react": "^18.2.0", "react-dom": "^18.2.0", diff --git a/ui/src/design-system/components/sort-control.tsx b/ui/src/design-system/components/sort-control.tsx index 87a18e2e0..bf1b6d9c1 100644 --- a/ui/src/design-system/components/sort-control.tsx +++ b/ui/src/design-system/components/sort-control.tsx @@ -1,6 +1,6 @@ import classNames from 'classnames' -import { ArrowUpDownIcon } from 'lucide-react' -import { buttonVariants, Select } from 'nova-ui-kit' +import { ArrowDownIcon, ArrowUpDownIcon } from 'lucide-react' +import { Button, buttonVariants, Select } from 'nova-ui-kit' import { STRING, translate } from 'utils/language' import { TableSortSettings } from './table/types' import { BasicTooltip } from './tooltip/basic-tooltip' @@ -16,33 +16,73 @@ export const SortControl = ({ columns, setSort, sort }: SortControlProps) => { ? columns.find((column) => column.sortField === sort.field) : undefined + const changeSortField = (field: string) => { + if (sort) { + setSort({ field, order: sort.order }) + } else { + setSort({ field, order: 'asc' }) + } + } + + const changeSortOrder = () => { + if (sort) { + setSort({ + field: sort.field, + order: sort.order === 'asc' ? 'desc' : 'asc', + }) + } + } + return ( - { - setSort({ field: value, order: 'asc' }) - }} - > - - - - {column ? column.name : translate(STRING.SORT_BY)} - - - - {columns - .filter((column) => column.sortField) - .map((column) => ( - - {column.name} - - ))} - - +
+ + + + {column ? column.name : translate(STRING.SORT_BY)} + {sort ? ( + + ) : null} + + + + {columns + .filter((column) => column.sortField) + .map((column) => ( + + {column.name} + + ))} + + + {sort ? ( + + + + ) : null} +
) } diff --git a/ui/src/utils/language.ts b/ui/src/utils/language.ts index cb54f81c7..9f107e02b 100644 --- a/ui/src/utils/language.ts +++ b/ui/src/utils/language.ts @@ -5,6 +5,7 @@ export enum STRING { BACK, CANCEL, CHANGE_IMAGE, + CHANGE_SORT_ORDER, CHOOSE_IMAGE, CLEAR_FILTERS, CLEAR, @@ -345,6 +346,7 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.BACK]: 'Back', [STRING.CANCEL]: 'Cancel', [STRING.CHANGE_IMAGE]: 'Change image', + [STRING.CHANGE_SORT_ORDER]: 'Change sort order', [STRING.CHOOSE_IMAGE]: 'Choose image', [STRING.CLEAR_FILTERS]: 'Clear filters', [STRING.CLEAR]: 'Clear', diff --git a/ui/yarn.lock b/ui/yarn.lock index 2492bfe7e..92b044a26 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -5002,7 +5002,7 @@ __metadata: leaflet: "npm:^1.9.3" lodash: "npm:^4.17.21" lucide-react: "npm:^1.0.1" - nova-ui-kit: "npm:^1.1.34" + nova-ui-kit: "npm:^1.1.36" plotly.js: "npm:^2.25.2" postcss: "npm:^8.4.47" prettier: "npm:2.8.4" @@ -10562,9 +10562,9 @@ __metadata: languageName: node linkType: hard -"nova-ui-kit@npm:^1.1.34": - version: 1.1.34 - resolution: "nova-ui-kit@npm:1.1.34" +"nova-ui-kit@npm:^1.1.36": + version: 1.1.36 + resolution: "nova-ui-kit@npm:1.1.36" dependencies: "@radix-ui/react-checkbox": "npm:^1.1.4" "@radix-ui/react-collapsible": "npm:^1.1.1" @@ -10584,7 +10584,7 @@ __metadata: react-dom: "npm:^18.3.1" tailwind-merge: "npm:^2.5.4" tailwindcss-animate: "npm:^1.0.7" - checksum: 4b3341ebebc715c79b5172870b30b33f65381bbf8d2acdcdda292e7be662592e32d188e9e33efd650c548211e45bef62efbb6af2ba464fb1eae37a31a02512db + checksum: c066ce1bc6d29339ee0596655135a9450bc55181eeb811970c93216889ef0361604897f3263905d2d0f01507f4296896111dff29d02770cd4da5f423aa46fef9 languageName: node linkType: hard From 917161c996478fefbb411b72ed8e71241fc47e0a Mon Sep 17 00:00:00 2001 From: Anna Viklund Date: Thu, 14 May 2026 15:52:01 +0200 Subject: [PATCH 04/13] feat: add sorting to projects view --- .../design-system/components/sort-control.tsx | 2 +- ui/src/pages/projects/projects.tsx | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/ui/src/design-system/components/sort-control.tsx b/ui/src/design-system/components/sort-control.tsx index bf1b6d9c1..b9887130f 100644 --- a/ui/src/design-system/components/sort-control.tsx +++ b/ui/src/design-system/components/sort-control.tsx @@ -42,7 +42,7 @@ export const SortControl = ({ columns, setSort, sort }: SortControlProps) => { buttonVariants({ size: 'small', variant: 'outline' }), 'w-auto' )} - hideIcon + hideIcon={!!sort} > {column ? column.name : translate(STRING.SORT_BY)} {sort ? ( diff --git a/ui/src/pages/projects/projects.tsx b/ui/src/pages/projects/projects.tsx index 323181bb3..9bdc13780 100644 --- a/ui/src/pages/projects/projects.tsx +++ b/ui/src/pages/projects/projects.tsx @@ -2,6 +2,7 @@ import { useProjects } from 'data-services/hooks/projects/useProjects' import { PageFooter } from 'design-system/components/page-footer/page-footer' import { PageHeader } from 'design-system/components/page-header/page-header' import { PaginationBar } from 'design-system/components/pagination-bar/pagination-bar' +import { SortControl } from 'design-system/components/sort-control' import * as Tabs from 'design-system/components/tabs/tabs' import { Button } from 'nova-ui-kit' import { NewProjectDialog } from 'pages/project-details/new-project-dialog' @@ -12,6 +13,7 @@ import { UserPermission } from 'utils/user/types' import { useUser } from 'utils/user/userContext' import { useUserInfo } from 'utils/user/userInfoContext' import { useSelectedView } from 'utils/useSelectedView' +import { useSort } from 'utils/useSort' import { ProjectGallery } from './project-gallery' export const TABS = { @@ -19,18 +21,24 @@ export const TABS = { ALL_PROJECTS: 'all-projects', } +const SORT_FIELDS = [ + { id: 'created_at', name: translate(STRING.FIELD_LABEL_CREATED_AT) }, + { id: 'updated_at', name: translate(STRING.FIELD_LABEL_UPDATED_AT) }, +] + export const Projects = () => { const { user } = useUser() const { userInfo } = useUserInfo() const { selectedView: selectedTab, setSelectedView: setSelectedTab } = useSelectedView(user.loggedIn ? TABS.MY_PROJECTS : TABS.ALL_PROJECTS) + const { sort, setSort } = useSort() const { pagination, setPage } = usePagination({ perPage: 40 }) const filters = user.loggedIn && selectedTab === TABS.MY_PROJECTS ? [{ field: 'user_id', value: userInfo?.id }] : [] const { projects, total, userPermissions, isLoading, isFetching, error } = - useProjects({ pagination, filters }) + useProjects({ pagination, filters, sort }) const canCreate = userPermissions?.includes(UserPermission.Create) return ( @@ -63,6 +71,14 @@ export const Projects = () => { ) : null} {canCreate ? : null} + ({ + ...field, + sortField: field.id, + }))} + setSort={setSort} + sort={sort} + /> {projects && projects.length === 0 && canCreate ? (
From 7082ce78eab15ad13212b8a52108181bdd3c0041 Mon Sep 17 00:00:00 2001 From: Anna Viklund Date: Thu, 14 May 2026 15:57:56 +0200 Subject: [PATCH 05/13] feat: make it possible to sort projects by name --- ami/main/api/views.py | 1 + ui/src/pages/projects/projects.tsx | 1 + 2 files changed, 2 insertions(+) diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 5a6d5aece..57664217c 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -155,6 +155,7 @@ class ProjectViewSet(DefaultViewSet, ProjectMixin): serializer_class = ProjectSerializer pagination_class = ProjectPagination permission_classes = [ObjectPermission] + ordering_fields = ["name", "created_at", "updated_at"] def get_queryset(self): qs: ProjectQuerySet = super().get_queryset() # type: ignore diff --git a/ui/src/pages/projects/projects.tsx b/ui/src/pages/projects/projects.tsx index 9bdc13780..9c1de4033 100644 --- a/ui/src/pages/projects/projects.tsx +++ b/ui/src/pages/projects/projects.tsx @@ -22,6 +22,7 @@ export const TABS = { } const SORT_FIELDS = [ + { id: 'name', name: translate(STRING.FIELD_LABEL_NAME) }, { id: 'created_at', name: translate(STRING.FIELD_LABEL_CREATED_AT) }, { id: 'updated_at', name: translate(STRING.FIELD_LABEL_UPDATED_AT) }, ] From 0ac2147326885c648dbdeead94d48fe172176f1b Mon Sep 17 00:00:00 2001 From: Anna Viklund Date: Thu, 14 May 2026 16:04:24 +0200 Subject: [PATCH 06/13] layout: cleanup --- ui/src/pages/jobs/jobs.tsx | 2 +- ui/src/pages/occurrences/occurrences.tsx | 2 +- ui/src/pages/project-details/new-project-dialog.tsx | 12 ++++++------ ui/src/pages/project/team/team-columns.tsx | 1 - ui/src/pages/project/team/team.tsx | 2 +- ui/src/utils/language.ts | 2 ++ 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/ui/src/pages/jobs/jobs.tsx b/ui/src/pages/jobs/jobs.tsx index c001cb859..e9696f18c 100644 --- a/ui/src/pages/jobs/jobs.tsx +++ b/ui/src/pages/jobs/jobs.tsx @@ -68,8 +68,8 @@ export const Jobs = () => { title={translate(STRING.NAV_ITEM_JOBS)} tooltip={translate(STRING.TOOLTIP_JOB)} > - {canCreate ? : null} + { to={APP_ROUTES.EXPORTS({ projectId: projectId as string })} > - Export + Export - + [] => [ { id: 'user', - sortField: 'name', name: translate(STRING.FIELD_LABEL_USER), renderCell: (item: Member) => ( diff --git a/ui/src/pages/project/team/team.tsx b/ui/src/pages/project/team/team.tsx index 6885a6065..1b04d61b6 100644 --- a/ui/src/pages/project/team/team.tsx +++ b/ui/src/pages/project/team/team.tsx @@ -24,7 +24,7 @@ export const Team = () => { }>() const { pagination, setPage } = usePagination() const { sort, setSort } = useSort({ - field: 'name', + field: 'created_at', order: 'asc', }) const { members, userPermissions, total, isLoading, isFetching, error } = diff --git a/ui/src/utils/language.ts b/ui/src/utils/language.ts index 9f107e02b..5e240cbd6 100644 --- a/ui/src/utils/language.ts +++ b/ui/src/utils/language.ts @@ -12,6 +12,7 @@ export enum STRING { COLLAPSE, CONFIRM, CONFIRMED, + CREATE_NEW, CURRENT_LOCATION, DELETE, DELETED, @@ -353,6 +354,7 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.COLLAPSE]: 'Collapse', [STRING.CONFIRM]: 'Confirm', [STRING.CONFIRMED]: 'Confirmed', + [STRING.CREATE_NEW]: 'Create new', [STRING.CURRENT_LOCATION]: 'Use current location', [STRING.DELETE]: 'Delete', [STRING.DELETED]: 'Deleted', From 5ba43b3b1b9f13be862af5fca77aabe5cab392c3 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 20 May 2026 19:00:39 -0700 Subject: [PATCH 07/13] feat: add composite indexes for project activity sorting Backs the project "recent activity" sort options. SourceImage is large in production (tens of millions of rows), so the indexes are built CONCURRENTLY in a non-atomic migration that clears statement_timeout for the build. Co-Authored-By: Claude --- .../0085_project_activity_sort_indexes.py | 39 +++++++++++++++++++ ami/main/models.py | 6 +++ 2 files changed, 45 insertions(+) create mode 100644 ami/main/migrations/0085_project_activity_sort_indexes.py diff --git a/ami/main/migrations/0085_project_activity_sort_indexes.py b/ami/main/migrations/0085_project_activity_sort_indexes.py new file mode 100644 index 000000000..ad5d7e104 --- /dev/null +++ b/ami/main/migrations/0085_project_activity_sort_indexes.py @@ -0,0 +1,39 @@ +from django.contrib.postgres.operations import AddIndexConcurrently +from django.db import migrations, models + + +class Migration(migrations.Migration): + """Add composite indexes that back the project "recent activity" sort options. + + Both tables are large in production (SourceImage in the tens of millions), + so the indexes are built CONCURRENTLY to avoid taking a write lock during + deploy. This requires a non-atomic migration. + + Building these indexes can take longer than a configured ``statement_timeout`` + (development sets 30s, see ``config/settings/local.py``; a production role may + set one too). ``CREATE INDEX CONCURRENTLY`` runs as a single statement and is + subject to that timeout, so we clear it for this connection before building. + """ + + atomic = False + + dependencies = [ + ("main", "0084_revoke_delete_job_from_roles"), + ] + + operations = [ + # Runtime SET overrides the startup "-c statement_timeout" option and + # persists for the rest of this (non-atomic) migration's connection. + migrations.RunSQL( + sql="SET statement_timeout = 0;", + reverse_sql=migrations.RunSQL.noop, + ), + AddIndexConcurrently( + model_name="sourceimage", + index=models.Index(fields=["project", "-timestamp"], name="main_source_proj_ts_desc_idx"), + ), + AddIndexConcurrently( + model_name="occurrence", + index=models.Index(fields=["project", "-updated_at"], name="occur_proj_updated_desc_idx"), + ), + ] diff --git a/ami/main/models.py b/ami/main/models.py index b30b4e645..e327187a5 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -2219,6 +2219,9 @@ class Meta: models.Index(fields=["deployment", "timestamp"]), models.Index(fields=["event", "timestamp"]), models.Index(fields=["timestamp"]), + # Supports sorting projects by their most recent capture timestamp + # (see ProjectViewSet ordering "last_capture_timestamp"). + models.Index(fields=["project", "-timestamp"], name="main_source_proj_ts_desc_idx"), ] @@ -3340,6 +3343,9 @@ class Meta: fields=["determination_id", "project_id", "event_id"], name="occur_det_proj_evt", ), + # Supports sorting projects by their most recently updated occurrence + # (see ProjectViewSet ordering "last_occurrence_updated_at"). + models.Index(fields=["project", "-updated_at"], name="occur_proj_updated_desc_idx"), ] From bd26d37fd4d0c784912221fdfb5a33eec8dca050 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 20 May 2026 19:00:46 -0700 Subject: [PATCH 08/13] feat: sort projects by recent observations, identifications and jobs Add ordering by most recent capture timestamp, occurrence update and job update. The aggregate fields are annotated as correlated subqueries only when that ordering is requested, so the default project list stays cheap. Co-Authored-By: Claude --- ami/main/api/views.py | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 57664217c..f00d9b18a 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -5,7 +5,7 @@ from django.contrib.postgres.search import TrigramSimilarity from django.core import exceptions from django.db import models -from django.db.models import Prefetch, Q +from django.db.models import OuterRef, Prefetch, Q, Subquery from django.db.models.functions import Coalesce from django.db.models.query import QuerySet from django.forms import BooleanField, CharField, IntegerField @@ -155,7 +155,14 @@ class ProjectViewSet(DefaultViewSet, ProjectMixin): serializer_class = ProjectSerializer pagination_class = ProjectPagination permission_classes = [ObjectPermission] - ordering_fields = ["name", "created_at", "updated_at"] + ordering_fields = [ + "name", + "created_at", + "updated_at", + "last_capture_timestamp", + "last_occurrence_updated_at", + "last_job_updated_at", + ] def get_queryset(self): qs: ProjectQuerySet = super().get_queryset() # type: ignore @@ -167,6 +174,32 @@ def get_queryset(self): raise PermissionDenied("You can only view your projects") if user: qs = qs.filter_by_user(user) + + # Annotate "recent activity" fields only when sorting by them, so the + # default list stays cheap (these are correlated subqueries over large + # related tables). Subqueries avoid the row fan-out that joined Max() + # annotations would cause when both are requested at once. + ordering = {field.lstrip("-") for field in self.request.query_params.get("ordering", "").split(",") if field} + if "last_capture_timestamp" in ordering: + qs = qs.annotate( + last_capture_timestamp=Subquery( + SourceImage.objects.filter(project=OuterRef("pk")).order_by("-timestamp").values("timestamp")[:1] + ) + ) + if "last_occurrence_updated_at" in ordering: + qs = qs.annotate( + last_occurrence_updated_at=Subquery( + Occurrence.objects.filter(project=OuterRef("pk")).order_by("-updated_at").values("updated_at")[:1] + ) + ) + if "last_job_updated_at" in ordering: + from ami.jobs.models import Job + + qs = qs.annotate( + last_job_updated_at=Subquery( + Job.objects.filter(project=OuterRef("pk")).order_by("-updated_at").values("updated_at")[:1] + ) + ) return qs def get_serializer_class(self): From 9eb26269987e2a4d2f5d13b6a8d6e840558e2ec1 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 20 May 2026 19:00:51 -0700 Subject: [PATCH 09/13] feat: support per-field default sort order in sort control A column can declare a defaultSortOrder that is applied when the field is first selected, so date-like fields (e.g. "Recent ...") open newest-first. Co-Authored-By: Claude --- ui/src/design-system/components/sort-control.tsx | 14 ++++++++------ ui/src/design-system/components/table/types.ts | 3 +++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/ui/src/design-system/components/sort-control.tsx b/ui/src/design-system/components/sort-control.tsx index b9887130f..d7e604b00 100644 --- a/ui/src/design-system/components/sort-control.tsx +++ b/ui/src/design-system/components/sort-control.tsx @@ -6,7 +6,12 @@ import { TableSortSettings } from './table/types' import { BasicTooltip } from './tooltip/basic-tooltip' interface SortControlProps { - columns: { id: string; name: string; sortField?: string }[] + columns: { + id: string + name: string + sortField?: string + defaultSortOrder?: TableSortSettings['order'] + }[] setSort: (sort?: TableSortSettings) => void sort?: TableSortSettings } @@ -17,11 +22,8 @@ export const SortControl = ({ columns, setSort, sort }: SortControlProps) => { : undefined const changeSortField = (field: string) => { - if (sort) { - setSort({ field, order: sort.order }) - } else { - setSort({ field, order: 'asc' }) - } + const selected = columns.find((column) => column.sortField === field) + setSort({ field, order: selected?.defaultSortOrder ?? sort?.order ?? 'asc' }) } const changeSortOrder = () => { diff --git a/ui/src/design-system/components/table/types.ts b/ui/src/design-system/components/table/types.ts index 88abb1834..f2a495e91 100644 --- a/ui/src/design-system/components/table/types.ts +++ b/ui/src/design-system/components/table/types.ts @@ -17,6 +17,9 @@ export interface TableColumn { name: string renderCell: (item: T, rowIndex: number, columnIndex: number) => JSX.Element sortField?: string + // Order applied when this field is first selected in the sort control. Useful + // for date-like fields (e.g. "Recent ...") that read better newest-first. + defaultSortOrder?: 'asc' | 'desc' sticky?: boolean styles?: { textAlign?: TextAlign From b5c99d66a390901add5d96af973f272e487f7a0f Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 20 May 2026 19:01:00 -0700 Subject: [PATCH 10/13] feat: add recent activity sort options to projects view Adds "Recent observations", "Recent identifications" and "Recent jobs" sort options, each defaulting to newest-first. Co-Authored-By: Claude --- ui/src/pages/projects/projects.tsx | 15 +++++++++++++++ ui/src/utils/language.ts | 6 ++++++ 2 files changed, 21 insertions(+) diff --git a/ui/src/pages/projects/projects.tsx b/ui/src/pages/projects/projects.tsx index 9c1de4033..6fb87442f 100644 --- a/ui/src/pages/projects/projects.tsx +++ b/ui/src/pages/projects/projects.tsx @@ -25,6 +25,21 @@ const SORT_FIELDS = [ { id: 'name', name: translate(STRING.FIELD_LABEL_NAME) }, { id: 'created_at', name: translate(STRING.FIELD_LABEL_CREATED_AT) }, { id: 'updated_at', name: translate(STRING.FIELD_LABEL_UPDATED_AT) }, + { + id: 'last_capture_timestamp', + name: translate(STRING.SORT_RECENT_OBSERVATIONS), + defaultSortOrder: 'desc' as const, + }, + { + id: 'last_occurrence_updated_at', + name: translate(STRING.SORT_RECENT_IDENTIFICATIONS), + defaultSortOrder: 'desc' as const, + }, + { + id: 'last_job_updated_at', + name: translate(STRING.SORT_RECENT_JOBS), + defaultSortOrder: 'desc' as const, + }, ] export const Projects = () => { diff --git a/ui/src/utils/language.ts b/ui/src/utils/language.ts index 5e240cbd6..ae77a03a4 100644 --- a/ui/src/utils/language.ts +++ b/ui/src/utils/language.ts @@ -323,6 +323,9 @@ export enum STRING { SET_PASSWORD, SETTINGS, SORT_BY, + SORT_RECENT_IDENTIFICATIONS, + SORT_RECENT_JOBS, + SORT_RECENT_OBSERVATIONS, STAGES, SUMMARY, TABLE_COLUMNS, @@ -702,6 +705,9 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.SET_PASSWORD]: 'Set password', [STRING.SETTINGS]: 'Settings', [STRING.SORT_BY]: 'Sort by', + [STRING.SORT_RECENT_IDENTIFICATIONS]: 'Recent identifications', + [STRING.SORT_RECENT_JOBS]: 'Recent jobs', + [STRING.SORT_RECENT_OBSERVATIONS]: 'Recent observations', [STRING.STAGES]: 'Stages', [STRING.SUMMARY]: 'Summary', [STRING.TABLE_COLUMNS]: 'Table columns', From 8721f938d36e27d0f7d2bfb64453b96a6c01ba99 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Thu, 21 May 2026 13:02:25 -0700 Subject: [PATCH 11/13] perf(projects): sort recent observations via deployment rollup Use Max("deployments__last_capture_timestamp") for the last_capture_timestamp ordering instead of a correlated subquery over SourceImage. The per-deployment timestamp is already denormalized, so this avoids scanning the multi-million row SourceImage table and matches the live max value for every project. With the subquery gone, the dedicated SourceImage (project, -timestamp) index is no longer needed, so drop it from the model and migration 0085. The occurrence index stays, since recent identifications still uses a subquery. Co-Authored-By: Claude --- ami/main/api/views.py | 15 ++++++--------- .../0085_project_activity_sort_indexes.py | 13 ++++--------- ami/main/models.py | 3 --- 3 files changed, 10 insertions(+), 21 deletions(-) diff --git a/ami/main/api/views.py b/ami/main/api/views.py index f00d9b18a..5af4a6d63 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -5,7 +5,7 @@ from django.contrib.postgres.search import TrigramSimilarity from django.core import exceptions from django.db import models -from django.db.models import OuterRef, Prefetch, Q, Subquery +from django.db.models import Max, OuterRef, Prefetch, Q, Subquery from django.db.models.functions import Coalesce from django.db.models.query import QuerySet from django.forms import BooleanField, CharField, IntegerField @@ -176,16 +176,13 @@ def get_queryset(self): qs = qs.filter_by_user(user) # Annotate "recent activity" fields only when sorting by them, so the - # default list stays cheap (these are correlated subqueries over large - # related tables). Subqueries avoid the row fan-out that joined Max() - # annotations would cause when both are requested at once. + # default list stays cheap. Each ordering annotates a single field, so + # the GROUP BY a Max() introduces never fans out across relations. ordering = {field.lstrip("-") for field in self.request.query_params.get("ordering", "").split(",") if field} if "last_capture_timestamp" in ordering: - qs = qs.annotate( - last_capture_timestamp=Subquery( - SourceImage.objects.filter(project=OuterRef("pk")).order_by("-timestamp").values("timestamp")[:1] - ) - ) + # Roll up the denormalized per-deployment timestamp rather than scanning + # the (very large) SourceImage table; the values match in practice. + qs = qs.annotate(last_capture_timestamp=Max("deployments__last_capture_timestamp")) if "last_occurrence_updated_at" in ordering: qs = qs.annotate( last_occurrence_updated_at=Subquery( diff --git a/ami/main/migrations/0085_project_activity_sort_indexes.py b/ami/main/migrations/0085_project_activity_sort_indexes.py index ad5d7e104..afa3bcf37 100644 --- a/ami/main/migrations/0085_project_activity_sort_indexes.py +++ b/ami/main/migrations/0085_project_activity_sort_indexes.py @@ -3,13 +3,12 @@ class Migration(migrations.Migration): - """Add composite indexes that back the project "recent activity" sort options. + """Add the composite index that backs the project "recent identifications" sort. - Both tables are large in production (SourceImage in the tens of millions), - so the indexes are built CONCURRENTLY to avoid taking a write lock during - deploy. This requires a non-atomic migration. + Occurrence is large in production, so the index is built CONCURRENTLY to avoid + taking a write lock during deploy. This requires a non-atomic migration. - Building these indexes can take longer than a configured ``statement_timeout`` + Building the index can take longer than a configured ``statement_timeout`` (development sets 30s, see ``config/settings/local.py``; a production role may set one too). ``CREATE INDEX CONCURRENTLY`` runs as a single statement and is subject to that timeout, so we clear it for this connection before building. @@ -28,10 +27,6 @@ class Migration(migrations.Migration): sql="SET statement_timeout = 0;", reverse_sql=migrations.RunSQL.noop, ), - AddIndexConcurrently( - model_name="sourceimage", - index=models.Index(fields=["project", "-timestamp"], name="main_source_proj_ts_desc_idx"), - ), AddIndexConcurrently( model_name="occurrence", index=models.Index(fields=["project", "-updated_at"], name="occur_proj_updated_desc_idx"), diff --git a/ami/main/models.py b/ami/main/models.py index e327187a5..a932542f7 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -2219,9 +2219,6 @@ class Meta: models.Index(fields=["deployment", "timestamp"]), models.Index(fields=["event", "timestamp"]), models.Index(fields=["timestamp"]), - # Supports sorting projects by their most recent capture timestamp - # (see ProjectViewSet ordering "last_capture_timestamp"). - models.Index(fields=["project", "-timestamp"], name="main_source_proj_ts_desc_idx"), ] From 19e8608d1fcdca4500bad63eddaa37ed3c390826 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Thu, 21 May 2026 21:43:12 -0700 Subject: [PATCH 12/13] fix(projects): make recent-captures sort live, null-safe and cheap to count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sort "recent captures" by a live per-project max capture time again, via a correlated subquery backed by a new (project, -timestamp) index on SourceImage (migration 0086). This keeps the sort accurate the moment captures land, rather than depending on the denormalized Deployment timestamp staying fresh. SourceImage.timestamp is nullable and DESC orders NULLs first, so the subquery excludes null timestamps explicitly — otherwise a single undated capture in a project masks its real most-recent capture and drops it to the bottom of the list. Verified the result now matches max(timestamp) for every project. Also strip the activity annotations from the pagination COUNT query (ProjectPagination.get_count): they don't affect the row count, and leaving them in made every paginated list run the subqueries again just to count. Co-Authored-By: Claude --- ami/main/api/views.py | 30 ++++++++++++---- .../0086_sourceimage_recent_capture_index.py | 36 +++++++++++++++++++ ami/main/models.py | 3 ++ 3 files changed, 63 insertions(+), 6 deletions(-) create mode 100644 ami/main/migrations/0086_sourceimage_recent_capture_index.py diff --git a/ami/main/api/views.py b/ami/main/api/views.py index 5af4a6d63..d79779074 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -5,7 +5,7 @@ from django.contrib.postgres.search import TrigramSimilarity from django.core import exceptions from django.db import models -from django.db.models import Max, OuterRef, Prefetch, Q, Subquery +from django.db.models import OuterRef, Prefetch, Q, Subquery from django.db.models.functions import Coalesce from django.db.models.query import QuerySet from django.forms import BooleanField, CharField, IntegerField @@ -145,6 +145,12 @@ class DefaultReadOnlyViewSet(DefaultViewSetMixin, viewsets.ReadOnlyModelViewSet) class ProjectPagination(LimitOffsetPaginationWithPermissions): default_limit = 40 + def get_count(self, queryset): + # The recent-activity orderings annotate correlated subqueries onto the + # queryset. They don't change the row count, so strip them (and ordering) + # before counting to keep the pagination COUNT query cheap. + return super().get_count(queryset.order_by().values("pk")) + class ProjectViewSet(DefaultViewSet, ProjectMixin): """ @@ -159,6 +165,9 @@ class ProjectViewSet(DefaultViewSet, ProjectMixin): "name", "created_at", "updated_at", + # The three below are not Project fields; get_queryset annotates them on + # demand (see below). last_capture_timestamp mirrors the DeploymentViewSet + # ordering of the same name, but is a per-project rollup of capture times. "last_capture_timestamp", "last_occurrence_updated_at", "last_job_updated_at", @@ -176,13 +185,22 @@ def get_queryset(self): qs = qs.filter_by_user(user) # Annotate "recent activity" fields only when sorting by them, so the - # default list stays cheap. Each ordering annotates a single field, so - # the GROUP BY a Max() introduces never fans out across relations. + # default list stays cheap. Each is a correlated subquery returning one + # row via a covering index, and only one is ever added per request. ordering = {field.lstrip("-") for field in self.request.query_params.get("ordering", "").split(",") if field} if "last_capture_timestamp" in ordering: - # Roll up the denormalized per-deployment timestamp rather than scanning - # the (very large) SourceImage table; the values match in practice. - qs = qs.annotate(last_capture_timestamp=Max("deployments__last_capture_timestamp")) + # Live max capture time per project (Index Only Scan on + # main_source_proj_ts_desc_idx); kept live rather than reading the + # denormalized Deployment field so the sort never lags ingestion. + # timestamp is nullable, and DESC sorts NULLs first, so exclude them + # explicitly — otherwise a single undated capture masks the real max. + qs = qs.annotate( + last_capture_timestamp=Subquery( + SourceImage.objects.filter(project=OuterRef("pk"), timestamp__isnull=False) + .order_by("-timestamp") + .values("timestamp")[:1] + ) + ) if "last_occurrence_updated_at" in ordering: qs = qs.annotate( last_occurrence_updated_at=Subquery( diff --git a/ami/main/migrations/0086_sourceimage_recent_capture_index.py b/ami/main/migrations/0086_sourceimage_recent_capture_index.py new file mode 100644 index 000000000..ebec4efe3 --- /dev/null +++ b/ami/main/migrations/0086_sourceimage_recent_capture_index.py @@ -0,0 +1,36 @@ +from django.contrib.postgres.operations import AddIndexConcurrently +from django.db import migrations, models + + +class Migration(migrations.Migration): + """Add the index backing the project "recent captures" sort. + + SourceImage is large in production (tens of millions of rows), so the index + is built CONCURRENTLY to avoid taking a write lock during deploy, which + requires a non-atomic migration. + + A concurrent build on a table this size can exceed a configured + ``statement_timeout`` (development sets 30s, see ``config/settings/local.py``; + a production role may set one too). ``CREATE INDEX CONCURRENTLY`` runs as a + single statement subject to that timeout, so clear it for this connection + before building. + """ + + atomic = False + + dependencies = [ + ("main", "0085_project_activity_sort_indexes"), + ] + + operations = [ + # Runtime SET overrides the startup "-c statement_timeout" option and + # persists for the rest of this (non-atomic) migration's connection. + migrations.RunSQL( + sql="SET statement_timeout = 0;", + reverse_sql=migrations.RunSQL.noop, + ), + AddIndexConcurrently( + model_name="sourceimage", + index=models.Index(fields=["project", "-timestamp"], name="main_source_proj_ts_desc_idx"), + ), + ] diff --git a/ami/main/models.py b/ami/main/models.py index a932542f7..c1616754f 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -2219,6 +2219,9 @@ class Meta: models.Index(fields=["deployment", "timestamp"]), models.Index(fields=["event", "timestamp"]), models.Index(fields=["timestamp"]), + # Backs the project "recent captures" sort: a per-project max(timestamp) + # lookup (see ProjectViewSet ordering "last_capture_timestamp"). + models.Index(fields=["project", "-timestamp"], name="main_source_proj_ts_desc_idx"), ] From 3ebf9a3506657647365a73169d94c92c168a4cad Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Thu, 21 May 2026 21:43:25 -0700 Subject: [PATCH 13/13] refactor(projects): use explicit sort labels for activity options Label the three recent-activity sort options for what they actually order by: "Recent captures" (SourceImage timestamp), "Occurrence updates" (Occurrence.updated_at) and "Jobs activity" (Job.updated_at). The previous "Recent observations / identifications / jobs" wording leaned on domain terms that don't map 1:1 to the underlying fields. Backend ordering keys unchanged. Co-Authored-By: Claude --- ui/src/pages/projects/projects.tsx | 6 +++--- ui/src/utils/language.ts | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ui/src/pages/projects/projects.tsx b/ui/src/pages/projects/projects.tsx index 6fb87442f..5e6485807 100644 --- a/ui/src/pages/projects/projects.tsx +++ b/ui/src/pages/projects/projects.tsx @@ -27,17 +27,17 @@ const SORT_FIELDS = [ { id: 'updated_at', name: translate(STRING.FIELD_LABEL_UPDATED_AT) }, { id: 'last_capture_timestamp', - name: translate(STRING.SORT_RECENT_OBSERVATIONS), + name: translate(STRING.SORT_RECENT_CAPTURES), defaultSortOrder: 'desc' as const, }, { id: 'last_occurrence_updated_at', - name: translate(STRING.SORT_RECENT_IDENTIFICATIONS), + name: translate(STRING.SORT_OCCURRENCE_UPDATES), defaultSortOrder: 'desc' as const, }, { id: 'last_job_updated_at', - name: translate(STRING.SORT_RECENT_JOBS), + name: translate(STRING.SORT_JOBS_ACTIVITY), defaultSortOrder: 'desc' as const, }, ] diff --git a/ui/src/utils/language.ts b/ui/src/utils/language.ts index ae77a03a4..11d7978fc 100644 --- a/ui/src/utils/language.ts +++ b/ui/src/utils/language.ts @@ -323,9 +323,9 @@ export enum STRING { SET_PASSWORD, SETTINGS, SORT_BY, - SORT_RECENT_IDENTIFICATIONS, - SORT_RECENT_JOBS, - SORT_RECENT_OBSERVATIONS, + SORT_JOBS_ACTIVITY, + SORT_OCCURRENCE_UPDATES, + SORT_RECENT_CAPTURES, STAGES, SUMMARY, TABLE_COLUMNS, @@ -705,9 +705,9 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.SET_PASSWORD]: 'Set password', [STRING.SETTINGS]: 'Settings', [STRING.SORT_BY]: 'Sort by', - [STRING.SORT_RECENT_IDENTIFICATIONS]: 'Recent identifications', - [STRING.SORT_RECENT_JOBS]: 'Recent jobs', - [STRING.SORT_RECENT_OBSERVATIONS]: 'Recent observations', + [STRING.SORT_JOBS_ACTIVITY]: 'Jobs activity', + [STRING.SORT_OCCURRENCE_UPDATES]: 'Occurrence updates', + [STRING.SORT_RECENT_CAPTURES]: 'Recent captures', [STRING.STAGES]: 'Stages', [STRING.SUMMARY]: 'Summary', [STRING.TABLE_COLUMNS]: 'Table columns',