diff --git a/actions/setup/js/handle_agent_failure.cjs b/actions/setup/js/handle_agent_failure.cjs index 264dbbce22a..d210f45ccee 100644 --- a/actions/setup/js/handle_agent_failure.cjs +++ b/actions/setup/js/handle_agent_failure.cjs @@ -6,7 +6,7 @@ const { sanitizeContent } = require("./sanitize_content.cjs"); const { getDetectionCautionAlert, getFooterAgentFailureIssueMessage, getFooterAgentFailureCommentMessage, generateXMLMarker } = require("./messages.cjs"); const { renderTemplate, renderTemplateFromFile, getPromptPath } = require("./messages_core.cjs"); const { getCurrentBranch } = require("./get_current_branch.cjs"); -const { createExpirationLine, generateFooterWithExpiration } = require("./ephemerals.cjs"); +const { createExpirationLine, extractExpirationDate, generateFooterWithExpiration } = require("./ephemerals.cjs"); const { MAX_SUB_ISSUES, getSubIssueCount } = require("./sub_issue_helpers.cjs"); const { formatMissingData, formatMissingTools } = require("./missing_info_formatter.cjs"); const { generateHistoryUrl } = require("./generate_history_link.cjs"); @@ -96,6 +96,201 @@ async function findPullRequestForCurrentBranch() { } } +/** + * Parse HTML comment metadata into key/value pairs. + * @param {string} body - Body text to inspect + * @param {string} markerKey - Marker key that must be present in the comment + * @returns {Record|null} Parsed metadata or null when not found + */ +function parseHTMLCommentMetadata(body, markerKey) { + if (!body) { + return null; + } + + for (const match of body.matchAll(//g)) { + const content = match[1].trim(); + if (!content.includes(`${markerKey}:`)) { + continue; + } + + /** @type {Record} */ + const metadata = {}; + const pairMatches = [...content.matchAll(/(?:^|,\s*)([a-zA-Z0-9_-]+):\s*/g)]; + for (let index = 0; index < pairMatches.length; index += 1) { + const pairMatch = pairMatches[index]; + const nextPairMatch = pairMatches[index + 1]; + const valueStart = (pairMatch.index || 0) + pairMatch[0].length; + const valueEnd = nextPairMatch ? nextPairMatch.index || content.length : content.length; + metadata[pairMatch[1]] = content.slice(valueStart, valueEnd).trim(); + } + + if (metadata[markerKey]) { + return metadata; + } + } + + return null; +} + +/** + * Build the stable category set used to match failure issues precisely. + * @param {Object} options - Active failure signals + * @returns {string[]} Sorted failure categories + */ +function buildFailureMatchCategories(options) { + const categories = []; + + if (options.isTimedOut) categories.push("timed_out"); + if (options.hasAssignmentErrors) categories.push("assignment_errors"); + if (options.hasAssignCopilotFailures) categories.push("assign_copilot_failures"); + if (options.hasCreateDiscussionErrors) categories.push("create_discussion_errors"); + if (options.hasCodePushFailures) categories.push("code_push_failures"); + if (options.hasRepoMemoryValidationErrors) categories.push("repo_memory_validation_errors"); + if (options.hasPushRepoMemoryFailure) categories.push("push_repo_memory_failure"); + if (options.hasMissingSafeOutputs) categories.push("missing_safe_outputs"); + if (options.hasReportIncomplete) categories.push("report_incomplete"); + if (options.hasMissingTool) categories.push("missing_tool"); + if (options.hasMissingData) categories.push("missing_data"); + if (options.hasCacheMissMisconfiguration) categories.push("cache_miss_misconfiguration"); + if (options.secretVerificationFailed) categories.push("secret_verification_failed"); + if (options.inferenceAccessError) categories.push("inference_access_error"); + if (options.mcpPolicyError) categories.push("mcp_policy_error"); + if (options.modelNotSupportedError) categories.push("model_not_supported_error"); + if (options.effectiveTokensRateLimitError) categories.push("effective_tokens_rate_limit_error"); + if (options.hasAppTokenMintingFailed) categories.push("app_token_minting_failed"); + if (options.hasLockdownCheckFailed) categories.push("lockdown_check_failed"); + if (options.hasStaleLockFileFailed) categories.push("stale_lock_file_failed"); + + if (options.agentConclusion === "failure" && !options.isTimedOut) { + categories.push("agent_failure"); + } + + return categories.sort(); +} + +/** + * Generate a precise failure-match marker for failure issue bodies. + * @param {Object} options - Marker options + * @param {string} options.workflowId - Workflow identifier + * @param {string} options.branch - Triggering branch + * @param {number|undefined} options.pullRequestNumber - Triggering pull request number + * @param {string[]} options.failureCategories - Sorted failure categories + * @returns {string} HTML comment marker + */ +function generateFailureMatchMarker(options) { + const { workflowId, branch, pullRequestNumber, failureCategories } = options; + const parts = ["gh-aw-failure-issue: true", `workflow_id: ${workflowId}`, `branch: ${branch || ""}`, `failure_categories: ${failureCategories.join("|")}`]; + + if (pullRequestNumber) { + parts.push(`pull_request: ${pullRequestNumber}`); + } + + return ``; +} + +/** + * Determine whether an existing issue body matches the current failure precisely. + * @param {string} body - Existing issue body + * @param {Object} options - Match criteria + * @param {string} options.workflowId - Workflow identifier + * @param {string} options.branch - Triggering branch + * @param {number|undefined} options.pullRequestNumber - Triggering pull request number + * @param {string[]} options.failureCategories - Sorted failure categories + * @returns {boolean} True when the issue body matches and is not expired + */ +function isReusableFailureIssue(body, options) { + if (!body) { + return false; + } + + const expirationDate = extractExpirationDate(body); + if (expirationDate && expirationDate.getTime() <= Date.now()) { + return false; + } + + const workflowMarker = parseHTMLCommentMetadata(body, "gh-aw-agentic-workflow"); + if (!workflowMarker || workflowMarker.workflow_id !== options.workflowId) { + return false; + } + + const failureMarker = parseHTMLCommentMetadata(body, "gh-aw-failure-issue"); + if (!failureMarker) { + return false; + } + + if ((failureMarker.workflow_id || "") !== options.workflowId) { + return false; + } + if ((failureMarker.branch || "") !== (options.branch || "")) { + return false; + } + + const expectedPullRequest = options.pullRequestNumber ? String(options.pullRequestNumber) : ""; + if ((failureMarker.pull_request || "") !== expectedPullRequest) { + return false; + } + + return (failureMarker.failure_categories || "") === options.failureCategories.join("|"); +} + +/** + * Find an existing open failure issue that exactly matches the current failure metadata. + * @param {Object} options - Search options + * @param {string} options.owner - Repository owner + * @param {string} options.repo - Repository name + * @param {string} options.issueTitle - Failure issue title + * @param {string} options.workflowId - Workflow identifier + * @param {string} options.branch - Triggering branch + * @param {number|undefined} options.pullRequestNumber - Triggering pull request number + * @param {string[]} options.failureCategories - Sorted failure categories + * @returns {Promise<{number: number, html_url: string} | null>} Matching issue or null + */ +async function findExistingFailureIssue(options) { + const { owner, repo, issueTitle, workflowId, branch, pullRequestNumber, failureCategories } = options; + const searchQuery = `repo:${owner}/${repo} is:issue is:open label:agentic-workflows in:title "${issueTitle}"`; + const perPage = 100; + + for (let page = 1; ; page += 1) { + const searchResult = await github.rest.search.issuesAndPullRequests({ + q: searchQuery, + per_page: perPage, + page, + }); + + for (const item of searchResult.data.items) { + let body = typeof item.body === "string" ? item.body : ""; + if (!body) { + const issueResult = await github.rest.issues.get({ + owner, + repo, + issue_number: item.number, + }); + body = issueResult.data.body || ""; + } + + if ( + isReusableFailureIssue(body, { + workflowId, + branch, + pullRequestNumber, + failureCategories, + }) + ) { + return { + number: item.number, + html_url: item.html_url, + }; + } + } + + if (searchResult.data.items.length < perPage) { + break; + } + } + + return null; +} + /** * Search for or create the parent issue for all agentic workflow failures * @param {number|null} previousParentNumber - Previous parent issue number if creating due to limit @@ -1706,6 +1901,7 @@ async function main() { // Try to find a pull request for the current branch const pullRequest = await findPullRequestForCurrentBranch(); + const currentBranch = getCurrentBranch(); // Generate history URL for linking to all failure issues created by this workflow const historyUrl = generateHistoryUrl({ @@ -1735,21 +1931,45 @@ async function main() { // Sanitize workflow name for title const sanitizedWorkflowName = sanitizeContent(workflowName, { maxLength: 100 }); const issueTitle = `[aw] ${sanitizedWorkflowName} failed`; + const failureCategories = buildFailureMatchCategories({ + agentConclusion, + isTimedOut, + hasAssignmentErrors, + hasAssignCopilotFailures, + hasCreateDiscussionErrors, + hasCodePushFailures, + hasRepoMemoryValidationErrors: repoMemoryValidationErrors.length > 0, + hasPushRepoMemoryFailure, + hasMissingSafeOutputs, + hasReportIncomplete, + hasMissingTool, + hasMissingData, + hasCacheMissMisconfiguration, + secretVerificationFailed: secretVerificationResult === "failed", + inferenceAccessError, + mcpPolicyError, + modelNotSupportedError, + effectiveTokensRateLimitError, + hasAppTokenMintingFailed, + hasLockdownCheckFailed, + hasStaleLockFileFailed, + }); - core.info(`Checking for existing issue with title: "${issueTitle}"`); - - // Search for existing open issue with this title and label - const searchQuery = `repo:${owner}/${repo} is:issue is:open label:agentic-workflows in:title "${issueTitle}"`; + core.info(`Checking for existing issue with precise metadata match for title: "${issueTitle}"`); try { - const searchResult = await github.rest.search.issuesAndPullRequests({ - q: searchQuery, - per_page: 1, + const existingIssue = await findExistingFailureIssue({ + owner, + repo, + issueTitle, + workflowId: workflowID, + branch: currentBranch, + pullRequestNumber: pullRequest?.number, + failureCategories, }); - if (searchResult.data.total_count > 0) { + if (existingIssue) { // Issue exists, add a comment - const existingIssue = searchResult.data.items[0]; core.info(`Found existing issue #${existingIssue.number}: ${existingIssue.html_url}`); // Read comment template @@ -1934,9 +2154,6 @@ async function main() { const issueTemplatePath = getPromptPath("agent_failure_issue.md"); const issueTemplate = fs.readFileSync(issueTemplatePath, "utf8"); - // Get current branch information - const currentBranch = getCurrentBranch(); - // Build assignment errors context let assignmentErrorsContext = ""; if (hasAssignmentErrors && assignmentErrors) { @@ -2086,12 +2303,18 @@ async function main() { historyUrl: historyUrl || undefined, }; const footer = getFooterAgentFailureIssueMessage(ctx); + const failureMatchMarker = generateFailureMatchMarker({ + workflowId: workflowID, + branch: currentBranch, + pullRequestNumber: pullRequest?.number, + failureCategories, + }); // Add expiration marker inside the quoted footer section using helper const footerWithExpires = generateFooterWithExpiration({ footerText: footer, expiresHours: actionFailureIssueExpiresHours, - suffix: `\n\n${generateXMLMarker(workflowName, runUrl)}`, + suffix: `\n\n${generateXMLMarker(workflowName, runUrl)}\n${failureMatchMarker}`, }); // Prepend detection caution alert (when present) so it appears first in the issue body diff --git a/actions/setup/js/handle_agent_failure.test.cjs b/actions/setup/js/handle_agent_failure.test.cjs index 48ca3fe124e..af1131f5391 100644 --- a/actions/setup/js/handle_agent_failure.test.cjs +++ b/actions/setup/js/handle_agent_failure.test.cjs @@ -82,6 +82,8 @@ describe("handle_agent_failure", () => { process.env.GH_AW_AGENT_CONCLUSION = "failure"; process.env.GH_AW_DETECTION_CONCLUSION = "warning"; process.env.GH_AW_DETECTION_REASON = "threat_detected"; + process.env.GITHUB_HEAD_REF = "feature/detection-caution"; + process.env.GITHUB_WORKSPACE = tmpDir; }); afterEach(() => { @@ -92,6 +94,8 @@ describe("handle_agent_failure", () => { delete process.env.GH_AW_AGENT_CONCLUSION; delete process.env.GH_AW_DETECTION_CONCLUSION; delete process.env.GH_AW_DETECTION_REASON; + delete process.env.GITHUB_HEAD_REF; + delete process.env.GITHUB_WORKSPACE; if (tmpDir && fs.existsSync(tmpDir)) { fs.rmSync(tmpDir, { recursive: true, force: true }); @@ -112,7 +116,16 @@ describe("handle_agent_failure", () => { return { data: { total_count: 1, - items: [{ number: 42, html_url: "https://github.com/owner/repo/issues/42" }], + items: [ + { + number: 42, + html_url: "https://github.com/owner/repo/issues/42", + body: + "> footer\n> - [x] expires on Jan 1, 2099, 12:00 AM UTC\n\n" + + "\n" + + "", + }, + ], }, }; }), @@ -178,6 +191,357 @@ describe("handle_agent_failure", () => { }); }); + describe("main() precise failure issue matching", () => { + const fs = require("fs"); + const path = require("path"); + const os = require("os"); + + /** @type {string} */ + let tmpDir; + /** @type {string} */ + let promptsDir; + + function buildExistingIssueBody({ branch, categories, expires = "2099-01-01T00:00:00.000Z", pullRequestNumber, workflowName = "Test Workflow" } = {}) { + const prPart = pullRequestNumber ? `, pull_request: ${pullRequestNumber}` : ""; + return ( + `> Generated from [${workflowName}](https://github.com/owner/repo/actions/runs/123456)\n` + + `> - [x] expires on Jan 1, 2099, 12:00 AM UTC\n\n` + + `\n` + + `` + ); + } + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "aw-handle-agent-failure-match-")); + promptsDir = path.join(tmpDir, "gh-aw", "prompts"); + fs.mkdirSync(promptsDir, { recursive: true }); + fs.writeFileSync(path.join(promptsDir, "agent_failure_comment.md"), "COMMENT TEMPLATE CONTENT"); + fs.writeFileSync(path.join(promptsDir, "agent_failure_issue.md"), "ISSUE TEMPLATE CONTENT"); + + process.env.RUNNER_TEMP = tmpDir; + process.env.GH_AW_WORKFLOW_NAME = "Test Workflow"; + process.env.GH_AW_WORKFLOW_ID = "test-workflow"; + process.env.GH_AW_RUN_URL = "https://github.com/owner/repo/actions/runs/123456"; + process.env.GH_AW_AGENT_CONCLUSION = "success"; + process.env.GITHUB_HEAD_REF = "feature/current"; + process.env.GITHUB_WORKSPACE = tmpDir; + }); + + afterEach(() => { + delete process.env.RUNNER_TEMP; + delete process.env.GH_AW_WORKFLOW_NAME; + delete process.env.GH_AW_WORKFLOW_ID; + delete process.env.GH_AW_RUN_URL; + delete process.env.GH_AW_AGENT_CONCLUSION; + delete process.env.GITHUB_HEAD_REF; + delete process.env.GITHUB_WORKSPACE; + if (tmpDir && fs.existsSync(tmpDir)) { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it("adds a comment only when the existing issue metadata matches exactly", async () => { + const createCommentMock = vi.fn(async () => ({ data: { id: 1001 } })); + const createIssueMock = vi.fn(); + + global.github = { + rest: { + search: { + issuesAndPullRequests: vi.fn(async ({ q }) => { + if (q.includes("is:pr")) { + return { data: { total_count: 0, items: [] } }; + } + return { + data: { + total_count: 1, + items: [ + { + number: 42, + html_url: "https://github.com/owner/repo/issues/42", + body: buildExistingIssueBody({ branch: "feature/current", categories: ["missing_safe_outputs"] }), + }, + ], + }, + }; + }), + }, + issues: { + create: createIssueMock, + createComment: createCommentMock, + }, + pulls: { get: vi.fn() }, + }, + graphql: vi.fn(), + }; + + await main(); + + expect(createCommentMock).toHaveBeenCalledOnce(); + expect(createIssueMock).not.toHaveBeenCalled(); + }); + + it("adds a comment when existing issue metadata contains commas in free-form values", async () => { + const createCommentMock = vi.fn(async () => ({ data: { id: 1001 } })); + const createIssueMock = vi.fn(); + + process.env.GH_AW_WORKFLOW_NAME = "Test Workflow, Retry"; + process.env.GITHUB_HEAD_REF = "feature/current,with-comma"; + + global.github = { + rest: { + search: { + issuesAndPullRequests: vi.fn(async ({ q }) => { + if (q.includes("is:pr")) { + return { data: { total_count: 0, items: [] } }; + } + return { + data: { + total_count: 1, + items: [ + { + number: 42, + html_url: "https://github.com/owner/repo/issues/42", + body: buildExistingIssueBody({ + workflowName: "Test Workflow, Retry", + branch: "feature/current,with-comma", + categories: ["missing_safe_outputs"], + }), + }, + ], + }, + }; + }), + }, + issues: { + create: createIssueMock, + createComment: createCommentMock, + }, + pulls: { get: vi.fn() }, + }, + graphql: vi.fn(), + }; + + await main(); + + expect(createCommentMock).toHaveBeenCalledOnce(); + expect(createIssueMock).not.toHaveBeenCalled(); + }); + + it("creates a new issue when an open issue with the same title has a different branch or category", async () => { + const createCommentMock = vi.fn(); + const createIssueMock = vi.fn(async ({ body }) => ({ + data: { number: 101, html_url: "https://github.com/owner/repo/issues/101", node_id: "I_123", body }, + })); + + global.github = { + rest: { + search: { + issuesAndPullRequests: vi.fn(async ({ q }) => { + if (q.includes("is:pr")) { + return { data: { total_count: 0, items: [] } }; + } + return { + data: { + total_count: 2, + items: [ + { + number: 42, + html_url: "https://github.com/owner/repo/issues/42", + body: buildExistingIssueBody({ branch: "feature/other", categories: ["missing_safe_outputs"] }), + }, + { + number: 43, + html_url: "https://github.com/owner/repo/issues/43", + body: buildExistingIssueBody({ branch: "feature/current", categories: ["agent_failure"] }), + }, + ], + }, + }; + }), + }, + issues: { + create: createIssueMock, + createComment: createCommentMock, + }, + pulls: { get: vi.fn() }, + }, + graphql: vi.fn(), + }; + + await main(); + + expect(createCommentMock).not.toHaveBeenCalled(); + expect(createIssueMock).toHaveBeenCalledOnce(); + expect(createIssueMock.mock.calls[0][0].body).toContain("gh-aw-failure-issue: true"); + expect(createIssueMock.mock.calls[0][0].body).toContain("branch: feature/current"); + expect(createIssueMock.mock.calls[0][0].body).toContain("failure_categories: missing_safe_outputs"); + }); + + it("creates a new issue instead of commenting on an expired issue", async () => { + const createCommentMock = vi.fn(); + const createIssueMock = vi.fn(async () => ({ + data: { number: 101, html_url: "https://github.com/owner/repo/issues/101", node_id: "I_123" }, + })); + + global.github = { + rest: { + search: { + issuesAndPullRequests: vi.fn(async ({ q }) => { + if (q.includes("is:pr")) { + return { data: { total_count: 0, items: [] } }; + } + return { + data: { + total_count: 1, + items: [ + { + number: 42, + html_url: "https://github.com/owner/repo/issues/42", + body: buildExistingIssueBody({ + branch: "feature/current", + categories: ["missing_safe_outputs"], + expires: "2000-01-01T00:00:00.000Z", + }), + }, + ], + }, + }; + }), + }, + issues: { + create: createIssueMock, + createComment: createCommentMock, + }, + pulls: { get: vi.fn() }, + }, + graphql: vi.fn(), + }; + + await main(); + + expect(createCommentMock).not.toHaveBeenCalled(); + expect(createIssueMock).toHaveBeenCalledOnce(); + }); + + it("continues searching later pages until it finds an exact metadata match", async () => { + const createCommentMock = vi.fn(async () => ({ data: { id: 1001 } })); + const createIssueMock = vi.fn(); + const searchMock = vi.fn(async ({ q, page }) => { + if (q.includes("is:pr")) { + return { data: { total_count: 0, items: [] } }; + } + + if (page === 1) { + return { + data: { + total_count: 101, + items: Array.from({ length: 100 }, (_, index) => ({ + number: index + 1, + html_url: `https://github.com/owner/repo/issues/${index + 1}`, + body: buildExistingIssueBody({ branch: `feature/other-${index + 1}`, categories: ["missing_safe_outputs"] }), + })), + }, + }; + } + + return { + data: { + total_count: 101, + items: [ + { + number: 101, + html_url: "https://github.com/owner/repo/issues/101", + body: buildExistingIssueBody({ branch: "feature/current", categories: ["missing_safe_outputs"] }), + }, + ], + }, + }; + }); + + global.github = { + rest: { + search: { + issuesAndPullRequests: searchMock, + }, + issues: { + create: createIssueMock, + createComment: createCommentMock, + }, + pulls: { get: vi.fn() }, + }, + graphql: vi.fn(), + }; + + await main(); + + expect(createCommentMock).toHaveBeenCalledOnce(); + expect(createIssueMock).not.toHaveBeenCalled(); + expect(searchMock).toHaveBeenCalledWith(expect.objectContaining({ page: 1 })); + expect(searchMock).toHaveBeenCalledWith(expect.objectContaining({ page: 2 })); + }); + + it("creates a new issue when the pull request metadata does not match exactly", async () => { + const createCommentMock = vi.fn(); + const createIssueMock = vi.fn(async ({ body }) => ({ + data: { number: 101, html_url: "https://github.com/owner/repo/issues/101", node_id: "I_123", body }, + })); + + global.github = { + rest: { + search: { + issuesAndPullRequests: vi.fn(async ({ q }) => { + if (q.includes("is:pr")) { + return { + data: { + total_count: 1, + items: [{ number: 123, html_url: "https://github.com/owner/repo/pull/123" }], + }, + }; + } + return { + data: { + total_count: 1, + items: [ + { + number: 42, + html_url: "https://github.com/owner/repo/issues/42", + body: buildExistingIssueBody({ + branch: "feature/current", + categories: ["missing_safe_outputs"], + pullRequestNumber: 999, + }), + }, + ], + }, + }; + }), + }, + issues: { + create: createIssueMock, + createComment: createCommentMock, + }, + pulls: { + get: vi.fn(async () => ({ + data: { + head: { sha: "abc123" }, + mergeable: true, + mergeable_state: "clean", + updated_at: "2026-05-18T00:00:00Z", + }, + })), + }, + }, + graphql: vi.fn(), + }; + + await main(); + + expect(createCommentMock).not.toHaveBeenCalled(); + expect(createIssueMock).toHaveBeenCalledOnce(); + expect(createIssueMock.mock.calls[0][0].body).toContain("pull_request: 123"); + }); + }); + describe("agent failure templates", () => { const fs = require("fs"); const path = require("path");