diff --git a/apps/sim/lib/sim-search/live/atlassian.test.ts b/apps/sim/lib/sim-search/live/atlassian.test.ts index 7b684c9d2bf..0c3fce42808 100644 --- a/apps/sim/lib/sim-search/live/atlassian.test.ts +++ b/apps/sim/lib/sim-search/live/atlassian.test.ts @@ -334,16 +334,32 @@ describe('Confluence plain search ranking', () => { describe('Jira plain search ranking', () => { const search = '/ex/jira/cloud/rest/api/3/search/jql' - /** Answers the all-terms query with `exact` and the summary query with `titled`, or its failure. */ - function jira(exact: { ids: string[]; more: boolean }, titled: string[] | (() => never)) { - const issues = (ids: string[]) => ids.map((key) => ({ key, fields: { summary: key } })) - const json = vi.fn(async (path: string, options?: { body?: { jql?: string } }) => { - if (path === '/oauth/token/accessible-resources') return [SITE] - if (path !== search) throw new Error(`Unexpected request: ${path}`) - if (options?.body?.jql?.includes('summary ~')) - return { issues: issues(typeof titled === 'function' ? titled() : titled) } - return { issues: issues(exact.ids), ...(exact.more ? { nextPageToken: 'page-2' } : {}) } - }) + type Issue = string | { key: string; summary: string } + /** + * Answers the all-terms query with `exact`, the all-terms summary query with `titled`, and the + * half-the-terms summary query with `half`, or their failures. + */ + function jira( + exact: { ids: string[]; more: boolean }, + titled: Issue[] | (() => never), + half: Issue[] | (() => never) = [] + ) { + const toRow = (row: Issue) => (typeof row === 'string' ? { key: row, summary: row } : row) + const issues = (rows: Issue[]) => + rows.map(toRow).map(({ key, summary }) => ({ key, fields: { summary } })) + const json = vi.fn( + async (path: string, options?: { body?: { jql?: string; maxResults?: number } }) => { + if (path === '/oauth/token/accessible-resources') return [SITE] + if (path !== search) throw new Error(`Unexpected request: ${path}`) + const jql = options?.body?.jql ?? '' + if (jql.includes('summary ~')) { + const rows = jql.includes(' OR ') ? half : titled + const picked = typeof rows === 'function' ? rows() : rows + return { issues: issues(picked).slice(0, options?.body?.maxResults) } + } + return { issues: issues(exact.ids), ...(exact.more ? { nextPageToken: 'page-2' } : {}) } + } + ) return { json, text: vi.fn(), bytes: vi.fn() } satisfies NativeClient } const query = { query: 'merged results pipelines', limit: 3, scopes: [] } @@ -411,4 +427,66 @@ describe('Jira plain search ranking', () => { expect(api.json).toHaveBeenCalledTimes(2) } }) + + it('leads with issues whose summary holds at least half the terms when none holds them all', async () => { + const api = jira( + overflowing, + [], + [ + { key: 'LOOSE-1', summary: 'Pipelines dashboard' }, + { key: 'HALF-1', summary: 'Merged results broke the dashboard' }, + ] + ) + const page = await searchAtlassian(api, 'jira', query) + expect(page.documents.map((document) => document.id)).toEqual(['HALF-1', 'NEW-3', 'NEW-2']) + expect(page.nextCursor).toBeUndefined() + }) + + it('keeps a two-term query to summaries holding every term', async () => { + const api = jira(overflowing, [], [{ key: 'HALF-1', summary: 'Merged pipelines' }]) + const page = await searchAtlassian(api, 'jira', { ...query, query: 'merged pipelines' }) + expect(page.documents.map((document) => document.id)).toEqual(['NEW-3', 'NEW-2', 'NEW-1']) + expect(api.json).toHaveBeenCalledTimes(3) + }) + + it('keeps the all-terms page and its cursor when the half-the-terms summary search fails', async () => { + const api = jira(overflowing, [], () => { + throw new NativeSearchError('rate_limited', 'Rate limited.', 30) + }) + const page = await searchAtlassian(api, 'jira', query) + expect(page.documents.map((document) => document.id)).toEqual(['NEW-3', 'NEW-2', 'NEW-1']) + expect(page.nextCursor).toBe('page-2') + }) + + it('counts a word and its plural as one term before allowing the half-the-terms fallback', async () => { + const api = jira(overflowing, [], [{ key: 'ONE-1', summary: 'Rotation schedule' }]) + const page = await searchAtlassian(api, 'jira', { ...query, query: 'token tokens rotation' }) + expect(page.documents.map((document) => document.id)).toEqual(['NEW-3', 'NEW-2', 'NEW-1']) + }) + + it('orders half-the-terms summaries by how many terms they hold', async () => { + const api = jira( + overflowing, + [], + [ + { key: 'TWO-1', summary: 'Merged results dashboard' }, + { key: 'THREE-1', summary: 'Merged results in release pipelines' }, + ] + ) + const page = await searchAtlassian(api, 'jira', { + ...query, + query: 'merged results release pipelines', + }) + expect(page.documents.map((document) => document.id)).toEqual(['THREE-1', 'TWO-1', 'NEW-3']) + }) + + it('finds half-the-terms summaries beyond the first page of pair matches', async () => { + const pairs = Array.from({ length: 10 }, (_, index) => ({ + key: `PAIR-${index}`, + summary: 'Pipelines dashboard', + })) + const api = jira(overflowing, [], [...pairs, { key: 'HALF-9', summary: 'Merged pipelines' }]) + const page = await searchAtlassian(api, 'jira', query) + expect(page.documents.map((document) => document.id)[0]).toBe('HALF-9') + }) }) diff --git a/apps/sim/lib/sim-search/live/atlassian.ts b/apps/sim/lib/sim-search/live/atlassian.ts index c72370cb563..5524763dbf8 100644 --- a/apps/sim/lib/sim-search/live/atlassian.ts +++ b/apps/sim/lib/sim-search/live/atlassian.ts @@ -12,8 +12,8 @@ import { providerText } from '@/lib/sim-search/live/text' import { confluenceTextClause, holdsHalfTheTerms, - jiraSummaryClause, jiraTextClause, + stemmedTerms, } from '@/lib/sim-search/live/text-match' import type { NativeClient, @@ -117,6 +117,13 @@ async function sites(client: NativeClient) { return array(await client.json('/oauth/token/accessible-resources', { memo: true })) } +/** + * Pair matches fetched for the half-the-terms summary fallback. Jira returns them newest first, + * not by relevance, and the page is then filtered to summaries holding enough terms, so it starts + * from a wider pool than the caller's limit or the any-term fallback's. + */ +const HALF_TERM_CANDIDATES = 50 + /** * One Atlassian site's page; `ranking` marks a page whose order or completeness differs from the * all-terms search. @@ -176,12 +183,16 @@ export async function searchAtlassian( const cloudId = string(site.id) const origin = string(site.url).replace(/\/$/, '') if (provider === 'jira') { - const searchIssues = async (jql: string, cursor?: string): Promise => { + /** One JQL page, each issue with its summary and raw fields. */ + const searchIssues = async ( + jql: string, + { cursor, maxResults = input.limit }: { cursor?: string; maxResults?: number } = {} + ) => { const data = object( await client.json(`/ex/jira/${segment(cloudId)}/rest/api/3/search/jql`, { body: { jql: scopeAtlassianQuery(jql, scope, order), - maxResults: input.limit, + maxResults, fields: ['summary', 'description', 'updated', 'creator', 'status'], ...(cursor ? { nextPageToken: cursor } : {}), }, @@ -189,42 +200,65 @@ export async function searchAtlassian( }) ) return { - documents: array(data.issues) - .filter((row) => { - const fields = object(row.fields) - return ( - !relevant || - relevant(`${string(fields.summary)}\n${textContent(fields.description)}`) - ) - }) - .map((row) => issue(row, cloudId, origin)), - excluded: false, + issues: array(data.issues).map((row) => { + const fields = object(row.fields) + return { + document: issue(row, cloudId, origin), + summary: string(fields.summary), + fields, + } + }), next: string(data.nextPageToken) || undefined, } } const exact = await searchIssues( input.native?.query || (text ? jiraTextClause(text, input.textMatch) : ''), - input.native?.cursor && single ? input.native.cursor : undefined + { cursor: input.native?.cursor && single ? input.native.cursor : undefined } ) + const exactPage: SitePage = { + documents: exact.issues + .filter( + ({ summary, fields }) => + !relevant || relevant(`${summary}\n${textContent(fields.description)}`) + ) + .map(({ document }) => document), + excluded: false, + next: exact.next, + } // JQL has no relevance order, so an all-terms match larger than a page comes back newest - // first; issues whose summary holds every term lead it instead. - if (!text || input.native || order || input.textMatch === 'any' || !exact.next) return exact - try { - const titled = await searchIssues(jiraSummaryClause(text)) - if (!titled.documents.length) return exact - const leading = new Set(titled.documents.map((document) => document.id)) - return { - documents: [ - ...titled.documents, - ...exact.documents.filter((document) => !leading.has(document.id)), - ].slice(0, input.limit), - excluded: false, - ranking: 'relevance', - } - } catch { - // A cancelled search stops here; any other failure keeps the all-terms page. - input.signal?.throwIfAborted() - return exact + // first; issues whose summary holds every term, or else at least half of three or more + // terms (most terms first), lead it instead. + if (!text || input.native || order || input.textMatch === 'any' || !exact.next) + return exactPage + const terms = stemmedTerms(text) + const [every, half] = await Promise.allSettled([ + searchIssues(jiraTextClause(text, 'all', 'summary')), + terms.count >= 3 + ? searchIssues(jiraTextClause(text, 'any', 'summary'), { + maxResults: HALF_TERM_CANDIDATES, + }) + : undefined, + ]) + // A cancelled search stops here; any other failure keeps the all-terms page. + input.signal?.throwIfAborted() + let titled = + every.status === 'fulfilled' ? every.value.issues.map(({ document }) => document) : [] + if (!titled.length && half.status === 'fulfilled' && half.value) { + titled = half.value.issues + .map(({ document, summary }) => ({ document, held: terms.held(summary) })) + .filter(({ held }) => held >= terms.required) + .sort((left, right) => right.held - left.held) + .map(({ document }) => document) + } + if (!titled.length) return exactPage + const leading = new Set(titled.map((document) => document.id)) + return { + documents: [ + ...titled, + ...exactPage.documents.filter((document) => !leading.has(document.id)), + ].slice(0, input.limit), + excluded: false, + ranking: 'relevance', } } const searchPages = async (clause: string, cursor?: string) => { @@ -289,7 +323,7 @@ export async function searchAtlassian( : '') + (pages.some((result) => result.ranking === 'relevance') ? provider === 'jira' - ? ' More issues contained every word than fit on one page, so issues whose summary holds every word come first; refine the query, or page with a native JQL query.' + ? ' More issues contained every word than fit on one page, so issues whose summary holds every word, or else most of them, come first; refine the query, or page with a native JQL query.' : ' More pages contained every word than fit on one page, so this page is ordered by Confluence relevance search; refine the query, or page with a native CQL query.' : '') + (pages.some((result) => result.ranking === 'unavailable') diff --git a/apps/sim/lib/sim-search/live/text-match.ts b/apps/sim/lib/sim-search/live/text-match.ts index 1b3ebe48f83..9f3f1bfa237 100644 --- a/apps/sim/lib/sim-search/live/text-match.ts +++ b/apps/sim/lib/sim-search/live/text-match.ts @@ -89,14 +89,19 @@ const escapeSearchPhrase = (value: string) => JSON.stringify(value.replace(/[+\-&|!(){}[\]^"~*?:\\/]/g, '\\$&')) /** - * The `text ~` clause for a plain query in Jira JQL. `any` ORs every pair of the query's longest - * terms, so a match contains at least two of them somewhere in the issue, or either term of a - * two-term query. Jira stems text search, so a word and its plural share one term. + * The clause for a plain query in Jira JQL, over all text fields or the summary alone. `any` ORs + * every pair of the query's longest terms, so a match contains at least two of them in that field, + * or either term of a two-term query. Jira stems text search, so a word and its plural share one + * term. */ -export function jiraTextClause(text: string, match: TextMatch = 'all'): string { +export function jiraTextClause( + text: string, + match: TextMatch = 'all', + field: 'text' | 'summary' = 'text' +): string { const terms = match === 'any' ? distinctTerms(text, singularKey) : [] - if (terms.length < 2) return `text ~ ${escapeSearchPhrase(text)}` - const clause = (term: string) => `text ~ ${escapeSearchPhrase(term)}` + if (terms.length < 2) return `${field} ~ ${escapeSearchPhrase(text)}` + const clause = (term: string) => `${field} ~ ${escapeSearchPhrase(term)}` const groups = terms.length === 2 ? terms.map(clause) @@ -106,14 +111,6 @@ export function jiraTextClause(text: string, match: TextMatch = 'all'): string { return `(${groups.join(' OR ')})` } -/** - * The `summary ~` clause for a plain query in Jira JQL: issues whose summary holds every word. - * JQL has no relevance order, so this stands in for a title boost. - */ -export function jiraSummaryClause(text: string): string { - return `summary ~ ${escapeSearchPhrase(text)}` -} - /** * The text clause for a plain query in Confluence CQL. `any` uses `siteSearch`, Confluence's own * relevance search, which ranks a page with the query's rarer words above the many pages sharing @@ -129,19 +126,35 @@ export function termCoverage(text: string, terms: readonly string[]): number { } /** - * A test for whether a text holds at least half the query's distinct terms. Query and text words - * are compared by their singular key, so a word and its regular plural count as one term as a - * stemmed search matches them; a term with punctuation inside it is matched as written. + * The query's distinct terms as a stemmed search counts them, and how many of them a text holds. + * Query and text words are compared by their singular key, so a word and its regular plural count + * as one term; a term with punctuation inside it is matched as written. */ -export function holdsHalfTheTerms(query: string): (text: string) => boolean { +export function stemmedTerms(query: string): { + count: number + /** At least half the terms, the minimum a partial match must hold. */ + required: number + held: (text: string) => number +} { const terms = distinctTerms(query, singularKey) const keys = terms.map(singularKey) - const required = Math.ceil(terms.length / 2) - return (text) => { - const words = new Set(textWords(text).map(singularKey)) - const present = terms.filter( - (term, index) => words.has(keys[index]) || findTermMatches(text, [term]).length > 0 - ) - return present.length >= required + return { + count: terms.length, + required: Math.ceil(terms.length / 2), + held: (text) => { + const words = new Set(textWords(text).map(singularKey)) + return terms.filter( + (term, index) => words.has(keys[index]) || findTermMatches(text, [term]).length > 0 + ).length + }, } } + +/** + * A test for whether a text holds at least half the query's distinct terms, as `stemmedTerms` + * counts them. + */ +export function holdsHalfTheTerms(query: string): (text: string) => boolean { + const { required, held } = stemmedTerms(query) + return (text) => held(text) >= required +}