Skip to content
Open
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
DESCRIPTION >
Precomputed organization-level KPIs for the org page. Rebuilt nightly by org_page_kpis_copy_pipe.
One row per organizationId. Used by org_page_kpis.pipe for cheap request-time lookups.

SCHEMA >
`organizationId` String,
`activeContributors` UInt64,
`activeContributorsPrevious` UInt64,
`maintainerRoles` UInt64,
`criticalProjects` UInt64,
`computedAt` DateTime

ENGINE ReplacingMergeTree
ENGINE_SORTING_KEY organizationId
ENGINE_VER computedAt
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
DESCRIPTION >
Precomputed per-org per-project metrics for the org page. Rebuilt nightly by org_page_projects_copy_pipe.
One row per (organizationId, segmentId). Used by org_page_projects.pipe.

SCHEMA >
`organizationId` String,
`segmentId` String,
`projectSlug` String,
`projectName` String,
`projectLogo` String,
`activityCount` UInt64,
`contributorCount` UInt64,
`computedAt` DateTime

ENGINE ReplacingMergeTree
ENGINE_SORTING_KEY organizationId, segmentId
ENGINE_VER computedAt
115 changes: 115 additions & 0 deletions services/libs/tinybird/pipes/org_page_activities_timeseries.pipe
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
DESCRIPTION >
Activity timeseries for a given organization, bucketed by granularity.
Filters to activities where the member belongs to the given org.

Comment on lines +1 to +3
TAGS "Organization page"
Comment on lines +1 to +4

NODE org_page_activities_bounds
SQL >
%
SELECT min(timestamp) AS actual_start_date, max(timestamp) AS actual_end_date
FROM activityRelations_deduplicated_cleaned_bucket_union
WHERE
organizationId = {{ String(orgId, '', description="Organization ID", required=True) }}
{% if defined(startDate) %}
AND timestamp >= {{ DateTime(startDate, description="Filter activity timestamp after") }}
{% end %}
{% if defined(endDate) %}
AND timestamp < {{ DateTime(endDate, description="Filter activity timestamp before") }}
{% end %}

NODE org_page_activities_timeseries_data
SQL >
%
SELECT
CASE
WHEN {{ String(granularity, 'monthly') }} = 'daily'
THEN toDate(addDays(bounds.actual_start_date, numbers.number))
WHEN {{ String(granularity, 'monthly') }} = 'weekly'
THEN toStartOfWeek(addDays(bounds.actual_start_date, numbers.number * 7))
WHEN {{ String(granularity, 'monthly') }} = 'monthly'
THEN toStartOfMonth(addMonths(bounds.actual_start_date, numbers.number))
WHEN {{ String(granularity, 'monthly') }} = 'quarterly'
THEN toStartOfQuarter(addMonths(bounds.actual_start_date, numbers.number * 3))
WHEN {{ String(granularity, 'monthly') }} = 'yearly'
THEN toStartOfYear(addYears(bounds.actual_start_date, numbers.number))
END AS startDate,
CASE
WHEN {{ String(granularity, 'monthly') }} = 'daily'
THEN toDate(addDays(bounds.actual_start_date, numbers.number))
WHEN {{ String(granularity, 'monthly') }} = 'weekly'
THEN
toDate(
toStartOfWeek(addDays(bounds.actual_start_date, numbers.number * 7))
+ INTERVAL 6 DAY
)
WHEN {{ String(granularity, 'monthly') }} = 'monthly'
THEN
toDate(
toStartOfMonth(addMonths(bounds.actual_start_date, numbers.number))
+ INTERVAL 1 MONTH
- INTERVAL 1 DAY
)
WHEN {{ String(granularity, 'monthly') }} = 'quarterly'
THEN
toDate(
toStartOfQuarter(addMonths(bounds.actual_start_date, numbers.number * 3))
+ INTERVAL 3 MONTH
- INTERVAL 1 DAY
)
WHEN {{ String(granularity, 'monthly') }} = 'yearly'
THEN
toDate(
toStartOfYear(addYears(bounds.actual_start_date, numbers.number))
+ INTERVAL 1 YEAR
- INTERVAL 1 DAY
)
END AS endDate,
count() AS activityCount
Comment thread
cursor[bot] marked this conversation as resolved.
Outdated
FROM numbers(1000) numbers
CROSS JOIN
(
SELECT
CASE
WHEN {{ String(granularity, 'monthly') }} = 'weekly'
THEN toStartOfWeek(actual_start_date)
WHEN {{ String(granularity, 'monthly') }} = 'monthly'
THEN toStartOfMonth(actual_start_date)
WHEN {{ String(granularity, 'monthly') }} = 'quarterly'
THEN toStartOfQuarter(actual_start_date)
WHEN {{ String(granularity, 'monthly') }} = 'yearly'
THEN toStartOfYear(actual_start_date)
ELSE actual_start_date
END AS actual_start_date,
actual_end_date
FROM org_page_activities_bounds
) bounds
LEFT JOIN
activityRelations_deduplicated_cleaned_bucket_union af
ON organizationId = {{ String(orgId, '') }}
AND CASE
WHEN {{ String(granularity, 'monthly') }} = 'daily'
THEN toDate(af.timestamp)
WHEN {{ String(granularity, 'monthly') }} = 'weekly'
THEN toStartOfWeek(af.timestamp)
WHEN {{ String(granularity, 'monthly') }} = 'monthly'
THEN toStartOfMonth(af.timestamp)
WHEN {{ String(granularity, 'monthly') }} = 'quarterly'
THEN toStartOfQuarter(af.timestamp)
WHEN {{ String(granularity, 'monthly') }} = 'yearly'
THEN toStartOfYear(af.timestamp)
END = CASE
WHEN {{ String(granularity, 'monthly') }} = 'daily'
THEN toDate(addDays(bounds.actual_start_date, numbers.number))
WHEN {{ String(granularity, 'monthly') }} = 'weekly'
THEN toStartOfWeek(addDays(bounds.actual_start_date, numbers.number * 7))
WHEN {{ String(granularity, 'monthly') }} = 'monthly'
THEN toStartOfMonth(addMonths(bounds.actual_start_date, numbers.number))
WHEN {{ String(granularity, 'monthly') }} = 'quarterly'
THEN toStartOfQuarter(addMonths(bounds.actual_start_date, numbers.number * 3))
WHEN {{ String(granularity, 'monthly') }} = 'yearly'
THEN toStartOfYear(addYears(bounds.actual_start_date, numbers.number))
END
WHERE startDate >= bounds.actual_start_date AND startDate < bounds.actual_end_date
GROUP BY startDate, endDate
ORDER BY startDate
Comment on lines +1 to +17
67 changes: 67 additions & 0 deletions services/libs/tinybird/pipes/org_page_contributors.pipe
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
DESCRIPTION >
Top contributors for a given organization leaderboard.
Returns members sorted by contribution count within the specified date range.

TAGS "Organization page"

NODE org_page_contributors_activity_aggregates
SQL >
%
{% if Boolean(count, false) %}
SELECT count(distinct memberId)
FROM activityRelations_deduplicated_cleaned_bucket_union
Comment on lines +13 to +18
WHERE
organizationId = {{ String(orgId, '', description="Organization ID", required=True) }}
{% if defined(startDate) %}
AND timestamp
>= {{ DateTime(startDate, description="Filter activity timestamp after") }}
{% end %}
{% if defined(endDate) %}
AND timestamp < {{ DateTime(endDate, description="Filter activity timestamp before") }}
{% end %}
{% else %}
SELECT
memberId,
count() as "contributionCount",
ROUND(COUNT(*) * 100.0 / SUM(COUNT(*)) OVER (), 2) as "contributionPercentage"
FROM activityRelations_deduplicated_cleaned_bucket_union
WHERE
organizationId = {{ String(orgId, '', description="Organization ID", required=True) }}
{% if defined(startDate) %}
AND timestamp
>= {{ DateTime(startDate, description="Filter activity timestamp after") }}
{% end %}
{% if defined(endDate) %}
AND timestamp < {{ DateTime(endDate, description="Filter activity timestamp before") }}
{% end %}
GROUP BY memberId
ORDER BY contributionCount DESC, memberId DESC
LIMIT {{ Int32(limit, 10) }}
OFFSET {{ Int32(offset, 0) }}
{% end %}

NODE org_page_contributors_leaderboard
SQL >
%
{% if Boolean(count, false) %}
SELECT count(distinct memberId) as count
FROM activityRelations_deduplicated_cleaned_bucket_union
WHERE
organizationId = {{ String(orgId, '') }}
{% if defined(startDate) %} AND timestamp >= {{ DateTime(startDate) }} {% end %}
{% if defined(endDate) %} AND timestamp < {{ DateTime(endDate) }} {% end %}
Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Count path duplicates expensive cross-segment table scan

Medium Severity

When count=true, the org_page_contributors_leaderboard node independently queries activityRelations_deduplicated_cleaned_bucket_union for the same count already computed (but unused) in org_page_contributors_activity_aggregates. Unlike contributors_leaderboard.pipe which references the project-scoped activities_filtered, this pipe scans the full unscoped cross-segment union (10-bucket UNION ALL) twice per request-time count call.

Additional Locations (1)
Fix in Cursor Fix in Web

Reviewed by Cursor Bugbot for commit b387268. Configure here.

{% else %}
SELECT
m.id,
m.avatar,
m.displayName,
m.githubHandleArray,
agg.contributionCount,
agg.contributionPercentage,
mr.roles
FROM members_sorted AS m ANY
INNER JOIN org_page_contributors_activity_aggregates agg ON agg.memberId = m.id
LEFT JOIN member_roles mr ON mr.memberId = m.id
WHERE m.id IN (SELECT memberId FROM org_page_contributors_activity_aggregates)
ORDER BY agg.contributionCount DESC
{% end %}
115 changes: 115 additions & 0 deletions services/libs/tinybird/pipes/org_page_contributors_timeseries.pipe
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
DESCRIPTION >
Contributor count timeseries for a given organization, bucketed by granularity.
Filters to activities where the member belongs to the given org.

TAGS "Organization page"

NODE org_page_contributors_bounds
SQL >
%
SELECT min(timestamp) AS actual_start_date, max(timestamp) AS actual_end_date
FROM activityRelations_deduplicated_cleaned_bucket_union
WHERE
organizationId = {{ String(orgId, '', description="Organization ID", required=True) }}
{% if defined(startDate) %}
AND timestamp >= {{ DateTime(startDate, description="Filter activity timestamp after") }}
{% end %}
{% if defined(endDate) %}
AND timestamp < {{ DateTime(endDate, description="Filter activity timestamp before") }}
{% end %}

NODE org_page_contributors_timeseries_data
SQL >
%
SELECT
CASE
WHEN {{ String(granularity, 'monthly') }} = 'daily'
THEN toDate(addDays(bounds.actual_start_date, numbers.number))
WHEN {{ String(granularity, 'monthly') }} = 'weekly'
THEN toStartOfWeek(addDays(bounds.actual_start_date, numbers.number * 7))
WHEN {{ String(granularity, 'monthly') }} = 'monthly'
THEN toStartOfMonth(addMonths(bounds.actual_start_date, numbers.number))
WHEN {{ String(granularity, 'monthly') }} = 'quarterly'
THEN toStartOfQuarter(addMonths(bounds.actual_start_date, numbers.number * 3))
WHEN {{ String(granularity, 'monthly') }} = 'yearly'
THEN toStartOfYear(addYears(bounds.actual_start_date, numbers.number))
END AS startDate,
CASE
WHEN {{ String(granularity, 'monthly') }} = 'daily'
THEN toDate(addDays(bounds.actual_start_date, numbers.number))
WHEN {{ String(granularity, 'monthly') }} = 'weekly'
THEN
toDate(
toStartOfWeek(addDays(bounds.actual_start_date, numbers.number * 7))
+ INTERVAL 6 DAY
)
WHEN {{ String(granularity, 'monthly') }} = 'monthly'
THEN
toDate(
toStartOfMonth(addMonths(bounds.actual_start_date, numbers.number))
+ INTERVAL 1 MONTH
- INTERVAL 1 DAY
)
WHEN {{ String(granularity, 'monthly') }} = 'quarterly'
THEN
toDate(
toStartOfQuarter(addMonths(bounds.actual_start_date, numbers.number * 3))
+ INTERVAL 3 MONTH
- INTERVAL 1 DAY
)
WHEN {{ String(granularity, 'monthly') }} = 'yearly'
THEN
toDate(
toStartOfYear(addYears(bounds.actual_start_date, numbers.number))
+ INTERVAL 1 YEAR
- INTERVAL 1 DAY
)
END AS endDate,
uniqExact(af.memberId) AS contributorCount
Comment thread
cursor[bot] marked this conversation as resolved.
Outdated
FROM numbers(1000) numbers
CROSS JOIN
(
SELECT
CASE
WHEN {{ String(granularity, 'monthly') }} = 'weekly'
THEN toStartOfWeek(actual_start_date)
WHEN {{ String(granularity, 'monthly') }} = 'monthly'
THEN toStartOfMonth(actual_start_date)
WHEN {{ String(granularity, 'monthly') }} = 'quarterly'
THEN toStartOfQuarter(actual_start_date)
WHEN {{ String(granularity, 'monthly') }} = 'yearly'
THEN toStartOfYear(actual_start_date)
ELSE actual_start_date
END AS actual_start_date,
actual_end_date
FROM org_page_contributors_bounds
) bounds
LEFT JOIN
activityRelations_deduplicated_cleaned_bucket_union af
ON organizationId = {{ String(orgId, '') }}
AND CASE
WHEN {{ String(granularity, 'monthly') }} = 'daily'
THEN toDate(af.timestamp)
WHEN {{ String(granularity, 'monthly') }} = 'weekly'
THEN toStartOfWeek(af.timestamp)
WHEN {{ String(granularity, 'monthly') }} = 'monthly'
THEN toStartOfMonth(af.timestamp)
WHEN {{ String(granularity, 'monthly') }} = 'quarterly'
THEN toStartOfQuarter(af.timestamp)
WHEN {{ String(granularity, 'monthly') }} = 'yearly'
THEN toStartOfYear(af.timestamp)
END = CASE
WHEN {{ String(granularity, 'monthly') }} = 'daily'
THEN toDate(addDays(bounds.actual_start_date, numbers.number))
WHEN {{ String(granularity, 'monthly') }} = 'weekly'
THEN toStartOfWeek(addDays(bounds.actual_start_date, numbers.number * 7))
WHEN {{ String(granularity, 'monthly') }} = 'monthly'
THEN toStartOfMonth(addMonths(bounds.actual_start_date, numbers.number))
WHEN {{ String(granularity, 'monthly') }} = 'quarterly'
THEN toStartOfQuarter(addMonths(bounds.actual_start_date, numbers.number * 3))
WHEN {{ String(granularity, 'monthly') }} = 'yearly'
THEN toStartOfYear(addYears(bounds.actual_start_date, numbers.number))
END
WHERE startDate >= bounds.actual_start_date AND startDate < bounds.actual_end_date
GROUP BY startDate, endDate
ORDER BY startDate
25 changes: 25 additions & 0 deletions services/libs/tinybird/pipes/org_page_kpis.pipe
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
DESCRIPTION >
Returns KPIs for a given organization from the precomputed org_page_kpis_copy_ds.
Includes trend calculations comparing current to previous 365-day period.

TAGS "Organization page"

NODE org_page_kpis_main
SQL >
%
SELECT
activeContributors,
if(
activeContributorsPrevious = 0, 0,
round(
(toInt64(activeContributors) - toInt64(activeContributorsPrevious))
/ activeContributorsPrevious * 100,
1
)
) AS activeContributorsTrend,
toInt64(activeContributors) - toInt64(activeContributorsPrevious) AS activeContributorsTrendAbsolute,
activeContributorsPrevious AS activeContributorsTrendPrevious,
maintainerRoles,
criticalProjects
FROM org_page_kpis_copy_ds FINAL
WHERE organizationId = {{ String(orgId, '', description="Organization ID", required=True) }}
Loading
Loading