From 1822130368d19099f377321882b7d20d3ce40c1b Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 15 Oct 2026 22:30:20 -0700 Subject: [PATCH 1/3] improvement(search): fall back to summaries holding half the terms of a longer query --- .../sim/lib/sim-search/live/atlassian.test.ts | 53 +++++++++++++-- apps/sim/lib/sim-search/live/atlassian.ts | 65 ++++++++++++------- apps/sim/lib/sim-search/live/text-match.ts | 25 ++++--- 3 files changed, 99 insertions(+), 44 deletions(-) diff --git a/apps/sim/lib/sim-search/live/atlassian.test.ts b/apps/sim/lib/sim-search/live/atlassian.test.ts index 7b684c9d2bf..8ee1ff71b50 100644 --- a/apps/sim/lib/sim-search/live/atlassian.test.ts +++ b/apps/sim/lib/sim-search/live/atlassian.test.ts @@ -334,14 +334,27 @@ describe('Confluence plain search ranking', () => { describe('Jira plain search ranking', () => { const search = '/ex/jira/cloud/rest/api/3/search/jql' - /** Answers the all-terms query with `exact` and the summary query with `titled`, or its failure. */ - function jira(exact: { ids: string[]; more: boolean }, titled: string[] | (() => never)) { - const issues = (ids: string[]) => ids.map((key) => ({ key, fields: { summary: key } })) + type Issue = string | { key: string; summary: string } + /** + * Answers the all-terms query with `exact`, the all-terms summary query with `titled`, and the + * half-the-terms summary query with `half`, or their failures. + */ + function jira( + exact: { ids: string[]; more: boolean }, + titled: Issue[] | (() => never), + half: Issue[] | (() => never) = [] + ) { + const issues = (rows: Issue[] | (() => never)) => + (typeof rows === 'function' ? rows() : rows).map((row) => + typeof row === 'string' + ? { key: row, fields: { summary: row } } + : { key: row.key, fields: { summary: row.summary } } + ) const json = vi.fn(async (path: string, options?: { body?: { jql?: string } }) => { if (path === '/oauth/token/accessible-resources') return [SITE] if (path !== search) throw new Error(`Unexpected request: ${path}`) - if (options?.body?.jql?.includes('summary ~')) - return { issues: issues(typeof titled === 'function' ? titled() : titled) } + const jql = options?.body?.jql ?? '' + if (jql.includes('summary ~')) return { issues: issues(jql.includes(' OR ') ? half : titled) } return { issues: issues(exact.ids), ...(exact.more ? { nextPageToken: 'page-2' } : {}) } }) return { json, text: vi.fn(), bytes: vi.fn() } satisfies NativeClient @@ -411,4 +424,34 @@ describe('Jira plain search ranking', () => { expect(api.json).toHaveBeenCalledTimes(2) } }) + + it('leads with issues whose summary holds at least half the terms when none holds them all', async () => { + const api = jira( + overflowing, + [], + [ + { key: 'HALF-1', summary: 'Merged results broke the pipelines' }, + { key: 'LOOSE-1', summary: 'Pipelines dashboard' }, + ] + ) + const page = await searchAtlassian(api, 'jira', query) + expect(page.documents.map((document) => document.id)).toEqual(['HALF-1', 'NEW-3', 'NEW-2']) + expect(page.nextCursor).toBeUndefined() + }) + + it('keeps a two-term query to summaries holding every term', async () => { + const api = jira(overflowing, [], [{ key: 'HALF-1', summary: 'Merged pipelines' }]) + const page = await searchAtlassian(api, 'jira', { ...query, query: 'merged pipelines' }) + expect(page.documents.map((document) => document.id)).toEqual(['NEW-3', 'NEW-2', 'NEW-1']) + expect(api.json).toHaveBeenCalledTimes(3) + }) + + it('keeps the all-terms page and its cursor when the half-the-terms summary search fails', async () => { + const api = jira(overflowing, [], () => { + throw new NativeSearchError('rate_limited', 'Rate limited.', 30) + }) + const page = await searchAtlassian(api, 'jira', query) + expect(page.documents.map((document) => document.id)).toEqual(['NEW-3', 'NEW-2', 'NEW-1']) + expect(page.nextCursor).toBe('page-2') + }) }) diff --git a/apps/sim/lib/sim-search/live/atlassian.ts b/apps/sim/lib/sim-search/live/atlassian.ts index c72370cb563..4982163fb26 100644 --- a/apps/sim/lib/sim-search/live/atlassian.ts +++ b/apps/sim/lib/sim-search/live/atlassian.ts @@ -12,8 +12,8 @@ import { providerText } from '@/lib/sim-search/live/text' import { confluenceTextClause, holdsHalfTheTerms, - jiraSummaryClause, jiraTextClause, + significantTerms, } from '@/lib/sim-search/live/text-match' import type { NativeClient, @@ -176,7 +176,12 @@ export async function searchAtlassian( const cloudId = string(site.id) const origin = string(site.url).replace(/\/$/, '') if (provider === 'jira') { - const searchIssues = async (jql: string, cursor?: string): Promise => { + /** One JQL page, keeping only issues whose summary and description pass `keep`. */ + const searchIssues = async ( + jql: string, + cursor?: string, + keep?: (summary: string, description: string) => boolean + ): Promise => { const data = object( await client.json(`/ex/jira/${segment(cloudId)}/rest/api/3/search/jql`, { body: { @@ -192,10 +197,7 @@ export async function searchAtlassian( documents: array(data.issues) .filter((row) => { const fields = object(row.fields) - return ( - !relevant || - relevant(`${string(fields.summary)}\n${textContent(fields.description)}`) - ) + return !keep || keep(string(fields.summary), textContent(fields.description)) }) .map((row) => issue(row, cloudId, origin)), excluded: false, @@ -204,27 +206,40 @@ export async function searchAtlassian( } const exact = await searchIssues( input.native?.query || (text ? jiraTextClause(text, input.textMatch) : ''), - input.native?.cursor && single ? input.native.cursor : undefined + input.native?.cursor && single ? input.native.cursor : undefined, + relevant && ((summary, description) => relevant(`${summary}\n${description}`)) ) // JQL has no relevance order, so an all-terms match larger than a page comes back newest - // first; issues whose summary holds every term lead it instead. + // first; issues whose summary holds every term, or else at least half of three or more + // terms, lead it instead. if (!text || input.native || order || input.textMatch === 'any' || !exact.next) return exact - try { - const titled = await searchIssues(jiraSummaryClause(text)) - if (!titled.documents.length) return exact - const leading = new Set(titled.documents.map((document) => document.id)) - return { - documents: [ - ...titled.documents, - ...exact.documents.filter((document) => !leading.has(document.id)), - ].slice(0, input.limit), - excluded: false, - ranking: 'relevance', - } - } catch { - // A cancelled search stops here; any other failure keeps the all-terms page. - input.signal?.throwIfAborted() - return exact + const [every, half] = await Promise.allSettled([ + searchIssues(jiraTextClause(text, 'all', 'summary')), + significantTerms(text).length >= 3 + ? searchIssues( + jiraTextClause(text, 'any', 'summary'), + undefined, + holdsHalfTheTerms(text) + ) + : undefined, + ]) + // A cancelled search stops here; any other failure keeps the all-terms page. + input.signal?.throwIfAborted() + const titled = + every.status === 'fulfilled' && every.value.documents.length + ? every.value.documents + : half.status === 'fulfilled' + ? (half.value?.documents ?? []) + : [] + if (!titled.length) return exact + const leading = new Set(titled.map((document) => document.id)) + return { + documents: [ + ...titled, + ...exact.documents.filter((document) => !leading.has(document.id)), + ].slice(0, input.limit), + excluded: false, + ranking: 'relevance', } } const searchPages = async (clause: string, cursor?: string) => { @@ -289,7 +304,7 @@ export async function searchAtlassian( : '') + (pages.some((result) => result.ranking === 'relevance') ? provider === 'jira' - ? ' More issues contained every word than fit on one page, so issues whose summary holds every word come first; refine the query, or page with a native JQL query.' + ? ' More issues contained every word than fit on one page, so issues whose summary holds the most query words come first; refine the query, or page with a native JQL query.' : ' More pages contained every word than fit on one page, so this page is ordered by Confluence relevance search; refine the query, or page with a native CQL query.' : '') + (pages.some((result) => result.ranking === 'unavailable') diff --git a/apps/sim/lib/sim-search/live/text-match.ts b/apps/sim/lib/sim-search/live/text-match.ts index 1b3ebe48f83..79e36791777 100644 --- a/apps/sim/lib/sim-search/live/text-match.ts +++ b/apps/sim/lib/sim-search/live/text-match.ts @@ -89,14 +89,19 @@ const escapeSearchPhrase = (value: string) => JSON.stringify(value.replace(/[+\-&|!(){}[\]^"~*?:\\/]/g, '\\$&')) /** - * The `text ~` clause for a plain query in Jira JQL. `any` ORs every pair of the query's longest - * terms, so a match contains at least two of them somewhere in the issue, or either term of a - * two-term query. Jira stems text search, so a word and its plural share one term. + * The clause for a plain query in Jira JQL, over all text fields or the summary alone. `any` ORs + * every pair of the query's longest terms, so a match contains at least two of them in that field, + * or either term of a two-term query. Jira stems text search, so a word and its plural share one + * term. */ -export function jiraTextClause(text: string, match: TextMatch = 'all'): string { +export function jiraTextClause( + text: string, + match: TextMatch = 'all', + field: 'text' | 'summary' = 'text' +): string { const terms = match === 'any' ? distinctTerms(text, singularKey) : [] - if (terms.length < 2) return `text ~ ${escapeSearchPhrase(text)}` - const clause = (term: string) => `text ~ ${escapeSearchPhrase(term)}` + if (terms.length < 2) return `${field} ~ ${escapeSearchPhrase(text)}` + const clause = (term: string) => `${field} ~ ${escapeSearchPhrase(term)}` const groups = terms.length === 2 ? terms.map(clause) @@ -106,14 +111,6 @@ export function jiraTextClause(text: string, match: TextMatch = 'all'): string { return `(${groups.join(' OR ')})` } -/** - * The `summary ~` clause for a plain query in Jira JQL: issues whose summary holds every word. - * JQL has no relevance order, so this stands in for a title boost. - */ -export function jiraSummaryClause(text: string): string { - return `summary ~ ${escapeSearchPhrase(text)}` -} - /** * The text clause for a plain query in Confluence CQL. `any` uses `siteSearch`, Confluence's own * relevance search, which ranks a page with the query's rarer words above the many pages sharing From e1a4ca2dbeeec7cc420731d0fc6bc501be3a86f6 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 15 Oct 2026 22:49:53 -0700 Subject: [PATCH 2/3] fix(search): count Jira half-term summaries by stemmed term, from a wider pool, most terms first --- .../sim/lib/sim-search/live/atlassian.test.ts | 56 +++++++++++-- apps/sim/lib/sim-search/live/atlassian.ts | 78 +++++++++++-------- apps/sim/lib/sim-search/live/text-match.ts | 30 ++++--- 3 files changed, 113 insertions(+), 51 deletions(-) diff --git a/apps/sim/lib/sim-search/live/atlassian.test.ts b/apps/sim/lib/sim-search/live/atlassian.test.ts index 8ee1ff71b50..e463a28723c 100644 --- a/apps/sim/lib/sim-search/live/atlassian.test.ts +++ b/apps/sim/lib/sim-search/live/atlassian.test.ts @@ -350,13 +350,21 @@ describe('Jira plain search ranking', () => { ? { key: row, fields: { summary: row } } : { key: row.key, fields: { summary: row.summary } } ) - const json = vi.fn(async (path: string, options?: { body?: { jql?: string } }) => { - if (path === '/oauth/token/accessible-resources') return [SITE] - if (path !== search) throw new Error(`Unexpected request: ${path}`) - const jql = options?.body?.jql ?? '' - if (jql.includes('summary ~')) return { issues: issues(jql.includes(' OR ') ? half : titled) } - return { issues: issues(exact.ids), ...(exact.more ? { nextPageToken: 'page-2' } : {}) } - }) + const json = vi.fn( + async (path: string, options?: { body?: { jql?: string; maxResults?: number } }) => { + if (path === '/oauth/token/accessible-resources') return [SITE] + if (path !== search) throw new Error(`Unexpected request: ${path}`) + const jql = options?.body?.jql ?? '' + if (jql.includes('summary ~')) + return { + issues: issues(jql.includes(' OR ') ? half : titled).slice( + 0, + options?.body?.maxResults + ), + } + return { issues: issues(exact.ids), ...(exact.more ? { nextPageToken: 'page-2' } : {}) } + } + ) return { json, text: vi.fn(), bytes: vi.fn() } satisfies NativeClient } const query = { query: 'merged results pipelines', limit: 3, scopes: [] } @@ -430,8 +438,8 @@ describe('Jira plain search ranking', () => { overflowing, [], [ - { key: 'HALF-1', summary: 'Merged results broke the pipelines' }, { key: 'LOOSE-1', summary: 'Pipelines dashboard' }, + { key: 'HALF-1', summary: 'Merged results broke the dashboard' }, ] ) const page = await searchAtlassian(api, 'jira', query) @@ -454,4 +462,36 @@ describe('Jira plain search ranking', () => { expect(page.documents.map((document) => document.id)).toEqual(['NEW-3', 'NEW-2', 'NEW-1']) expect(page.nextCursor).toBe('page-2') }) + + it('counts a word and its plural as one term before allowing the half-the-terms fallback', async () => { + const api = jira(overflowing, [], [{ key: 'ONE-1', summary: 'Rotation schedule' }]) + const page = await searchAtlassian(api, 'jira', { ...query, query: 'token tokens rotation' }) + expect(page.documents.map((document) => document.id)).toEqual(['NEW-3', 'NEW-2', 'NEW-1']) + }) + + it('orders half-the-terms summaries by how many terms they hold', async () => { + const api = jira( + overflowing, + [], + [ + { key: 'TWO-1', summary: 'Merged results dashboard' }, + { key: 'THREE-1', summary: 'Merged results in release pipelines' }, + ] + ) + const page = await searchAtlassian(api, 'jira', { + ...query, + query: 'merged results release pipelines', + }) + expect(page.documents.map((document) => document.id)).toEqual(['THREE-1', 'TWO-1', 'NEW-3']) + }) + + it('finds half-the-terms summaries beyond the first page of pair matches', async () => { + const pairs = Array.from({ length: 10 }, (_, index) => ({ + key: `PAIR-${index}`, + summary: 'Pipelines dashboard', + })) + const api = jira(overflowing, [], [...pairs, { key: 'HALF-9', summary: 'Merged pipelines' }]) + const page = await searchAtlassian(api, 'jira', query) + expect(page.documents.map((document) => document.id)[0]).toBe('HALF-9') + }) }) diff --git a/apps/sim/lib/sim-search/live/atlassian.ts b/apps/sim/lib/sim-search/live/atlassian.ts index 4982163fb26..f4f3e79721e 100644 --- a/apps/sim/lib/sim-search/live/atlassian.ts +++ b/apps/sim/lib/sim-search/live/atlassian.ts @@ -13,7 +13,7 @@ import { confluenceTextClause, holdsHalfTheTerms, jiraTextClause, - significantTerms, + stemmedTerms, } from '@/lib/sim-search/live/text-match' import type { NativeClient, @@ -117,6 +117,12 @@ async function sites(client: NativeClient) { return array(await client.json('/oauth/token/accessible-resources', { memo: true })) } +/** + * Pair matches fetched for the half-the-terms summary fallback; the page is filtered to summaries + * holding enough terms, so it starts from a wider pool than the caller's limit. + */ +const HALF_TERM_CANDIDATES = 50 + /** * One Atlassian site's page; `ranking` marks a page whose order or completeness differs from the * all-terms search. @@ -176,17 +182,13 @@ export async function searchAtlassian( const cloudId = string(site.id) const origin = string(site.url).replace(/\/$/, '') if (provider === 'jira') { - /** One JQL page, keeping only issues whose summary and description pass `keep`. */ - const searchIssues = async ( - jql: string, - cursor?: string, - keep?: (summary: string, description: string) => boolean - ): Promise => { + /** One JQL page, each issue with its summary and description. */ + const searchIssues = async (jql: string, cursor?: string, maxResults = input.limit) => { const data = object( await client.json(`/ex/jira/${segment(cloudId)}/rest/api/3/search/jql`, { body: { jql: scopeAtlassianQuery(jql, scope, order), - maxResults: input.limit, + maxResults, fields: ['summary', 'description', 'updated', 'creator', 'status'], ...(cursor ? { nextPageToken: cursor } : {}), }, @@ -194,49 +196,61 @@ export async function searchAtlassian( }) ) return { - documents: array(data.issues) - .filter((row) => { - const fields = object(row.fields) - return !keep || keep(string(fields.summary), textContent(fields.description)) - }) - .map((row) => issue(row, cloudId, origin)), - excluded: false, + issues: array(data.issues).map((row) => { + const fields = object(row.fields) + return { + document: issue(row, cloudId, origin), + summary: string(fields.summary), + description: textContent(fields.description), + } + }), next: string(data.nextPageToken) || undefined, } } const exact = await searchIssues( input.native?.query || (text ? jiraTextClause(text, input.textMatch) : ''), - input.native?.cursor && single ? input.native.cursor : undefined, - relevant && ((summary, description) => relevant(`${summary}\n${description}`)) + input.native?.cursor && single ? input.native.cursor : undefined ) + const exactPage: SitePage = { + documents: exact.issues + .filter( + ({ summary, description }) => !relevant || relevant(`${summary}\n${description}`) + ) + .map(({ document }) => document), + excluded: false, + next: exact.next, + } // JQL has no relevance order, so an all-terms match larger than a page comes back newest // first; issues whose summary holds every term, or else at least half of three or more - // terms, lead it instead. - if (!text || input.native || order || input.textMatch === 'any' || !exact.next) return exact + // terms (most terms first), lead it instead. + if (!text || input.native || order || input.textMatch === 'any' || !exact.next) + return exactPage + const terms = stemmedTerms(text) const [every, half] = await Promise.allSettled([ searchIssues(jiraTextClause(text, 'all', 'summary')), - significantTerms(text).length >= 3 - ? searchIssues( - jiraTextClause(text, 'any', 'summary'), - undefined, - holdsHalfTheTerms(text) - ) + terms.count >= 3 + ? searchIssues(jiraTextClause(text, 'any', 'summary'), undefined, HALF_TERM_CANDIDATES) : undefined, ]) // A cancelled search stops here; any other failure keeps the all-terms page. input.signal?.throwIfAborted() + const required = Math.ceil(terms.count / 2) const titled = - every.status === 'fulfilled' && every.value.documents.length - ? every.value.documents - : half.status === 'fulfilled' - ? (half.value?.documents ?? []) + every.status === 'fulfilled' && every.value.issues.length + ? every.value.issues.map(({ document }) => document) + : half.status === 'fulfilled' && half.value + ? half.value.issues + .map(({ document, summary }) => ({ document, held: terms.held(summary) })) + .filter(({ held }) => held >= required) + .sort((left, right) => right.held - left.held) + .map(({ document }) => document) : [] - if (!titled.length) return exact + if (!titled.length) return exactPage const leading = new Set(titled.map((document) => document.id)) return { documents: [ ...titled, - ...exact.documents.filter((document) => !leading.has(document.id)), + ...exactPage.documents.filter((document) => !leading.has(document.id)), ].slice(0, input.limit), excluded: false, ranking: 'relevance', @@ -304,7 +318,7 @@ export async function searchAtlassian( : '') + (pages.some((result) => result.ranking === 'relevance') ? provider === 'jira' - ? ' More issues contained every word than fit on one page, so issues whose summary holds the most query words come first; refine the query, or page with a native JQL query.' + ? ' More issues contained every word than fit on one page, so issues whose summary holds every word, or else most of them, come first; refine the query, or page with a native JQL query.' : ' More pages contained every word than fit on one page, so this page is ordered by Confluence relevance search; refine the query, or page with a native CQL query.' : '') + (pages.some((result) => result.ranking === 'unavailable') diff --git a/apps/sim/lib/sim-search/live/text-match.ts b/apps/sim/lib/sim-search/live/text-match.ts index 79e36791777..d956efd0fac 100644 --- a/apps/sim/lib/sim-search/live/text-match.ts +++ b/apps/sim/lib/sim-search/live/text-match.ts @@ -126,19 +126,27 @@ export function termCoverage(text: string, terms: readonly string[]): number { } /** - * A test for whether a text holds at least half the query's distinct terms. Query and text words - * are compared by their singular key, so a word and its regular plural count as one term as a - * stemmed search matches them; a term with punctuation inside it is matched as written. + * The query's distinct terms as a stemmed search counts them, and how many of them a text holds. + * Query and text words are compared by their singular key, so a word and its regular plural count + * as one term; a term with punctuation inside it is matched as written. */ -export function holdsHalfTheTerms(query: string): (text: string) => boolean { +export function stemmedTerms(query: string): { count: number; held: (text: string) => number } { const terms = distinctTerms(query, singularKey) const keys = terms.map(singularKey) - const required = Math.ceil(terms.length / 2) - return (text) => { - const words = new Set(textWords(text).map(singularKey)) - const present = terms.filter( - (term, index) => words.has(keys[index]) || findTermMatches(text, [term]).length > 0 - ) - return present.length >= required + return { + count: terms.length, + held: (text) => { + const words = new Set(textWords(text).map(singularKey)) + return terms.filter( + (term, index) => words.has(keys[index]) || findTermMatches(text, [term]).length > 0 + ).length + }, } } + +/** A test for whether a text holds at least half the query's distinct terms, as `stemmedTerms` counts them. */ +export function holdsHalfTheTerms(query: string): (text: string) => boolean { + const { count, held } = stemmedTerms(query) + const required = Math.ceil(count / 2) + return (text) => held(text) >= required +} From 9191e3fd3c76ae114ff4f288af4fb99eb542f64b Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 15 Oct 2026 22:56:43 -0700 Subject: [PATCH 3/3] improvement(search): share the half-terms threshold and convert Jira descriptions only when filtered --- .../sim/lib/sim-search/live/atlassian.test.ts | 21 ++++----- apps/sim/lib/sim-search/live/atlassian.ts | 43 +++++++++++-------- apps/sim/lib/sim-search/live/text-match.ts | 16 +++++-- 3 files changed, 44 insertions(+), 36 deletions(-) diff --git a/apps/sim/lib/sim-search/live/atlassian.test.ts b/apps/sim/lib/sim-search/live/atlassian.test.ts index e463a28723c..0c3fce42808 100644 --- a/apps/sim/lib/sim-search/live/atlassian.test.ts +++ b/apps/sim/lib/sim-search/live/atlassian.test.ts @@ -344,24 +344,19 @@ describe('Jira plain search ranking', () => { titled: Issue[] | (() => never), half: Issue[] | (() => never) = [] ) { - const issues = (rows: Issue[] | (() => never)) => - (typeof rows === 'function' ? rows() : rows).map((row) => - typeof row === 'string' - ? { key: row, fields: { summary: row } } - : { key: row.key, fields: { summary: row.summary } } - ) + const toRow = (row: Issue) => (typeof row === 'string' ? { key: row, summary: row } : row) + const issues = (rows: Issue[]) => + rows.map(toRow).map(({ key, summary }) => ({ key, fields: { summary } })) const json = vi.fn( async (path: string, options?: { body?: { jql?: string; maxResults?: number } }) => { if (path === '/oauth/token/accessible-resources') return [SITE] if (path !== search) throw new Error(`Unexpected request: ${path}`) const jql = options?.body?.jql ?? '' - if (jql.includes('summary ~')) - return { - issues: issues(jql.includes(' OR ') ? half : titled).slice( - 0, - options?.body?.maxResults - ), - } + if (jql.includes('summary ~')) { + const rows = jql.includes(' OR ') ? half : titled + const picked = typeof rows === 'function' ? rows() : rows + return { issues: issues(picked).slice(0, options?.body?.maxResults) } + } return { issues: issues(exact.ids), ...(exact.more ? { nextPageToken: 'page-2' } : {}) } } ) diff --git a/apps/sim/lib/sim-search/live/atlassian.ts b/apps/sim/lib/sim-search/live/atlassian.ts index f4f3e79721e..5524763dbf8 100644 --- a/apps/sim/lib/sim-search/live/atlassian.ts +++ b/apps/sim/lib/sim-search/live/atlassian.ts @@ -118,8 +118,9 @@ async function sites(client: NativeClient) { } /** - * Pair matches fetched for the half-the-terms summary fallback; the page is filtered to summaries - * holding enough terms, so it starts from a wider pool than the caller's limit. + * Pair matches fetched for the half-the-terms summary fallback. Jira returns them newest first, + * not by relevance, and the page is then filtered to summaries holding enough terms, so it starts + * from a wider pool than the caller's limit or the any-term fallback's. */ const HALF_TERM_CANDIDATES = 50 @@ -182,8 +183,11 @@ export async function searchAtlassian( const cloudId = string(site.id) const origin = string(site.url).replace(/\/$/, '') if (provider === 'jira') { - /** One JQL page, each issue with its summary and description. */ - const searchIssues = async (jql: string, cursor?: string, maxResults = input.limit) => { + /** One JQL page, each issue with its summary and raw fields. */ + const searchIssues = async ( + jql: string, + { cursor, maxResults = input.limit }: { cursor?: string; maxResults?: number } = {} + ) => { const data = object( await client.json(`/ex/jira/${segment(cloudId)}/rest/api/3/search/jql`, { body: { @@ -201,7 +205,7 @@ export async function searchAtlassian( return { document: issue(row, cloudId, origin), summary: string(fields.summary), - description: textContent(fields.description), + fields, } }), next: string(data.nextPageToken) || undefined, @@ -209,12 +213,13 @@ export async function searchAtlassian( } const exact = await searchIssues( input.native?.query || (text ? jiraTextClause(text, input.textMatch) : ''), - input.native?.cursor && single ? input.native.cursor : undefined + { cursor: input.native?.cursor && single ? input.native.cursor : undefined } ) const exactPage: SitePage = { documents: exact.issues .filter( - ({ summary, description }) => !relevant || relevant(`${summary}\n${description}`) + ({ summary, fields }) => + !relevant || relevant(`${summary}\n${textContent(fields.description)}`) ) .map(({ document }) => document), excluded: false, @@ -229,22 +234,22 @@ export async function searchAtlassian( const [every, half] = await Promise.allSettled([ searchIssues(jiraTextClause(text, 'all', 'summary')), terms.count >= 3 - ? searchIssues(jiraTextClause(text, 'any', 'summary'), undefined, HALF_TERM_CANDIDATES) + ? searchIssues(jiraTextClause(text, 'any', 'summary'), { + maxResults: HALF_TERM_CANDIDATES, + }) : undefined, ]) // A cancelled search stops here; any other failure keeps the all-terms page. input.signal?.throwIfAborted() - const required = Math.ceil(terms.count / 2) - const titled = - every.status === 'fulfilled' && every.value.issues.length - ? every.value.issues.map(({ document }) => document) - : half.status === 'fulfilled' && half.value - ? half.value.issues - .map(({ document, summary }) => ({ document, held: terms.held(summary) })) - .filter(({ held }) => held >= required) - .sort((left, right) => right.held - left.held) - .map(({ document }) => document) - : [] + let titled = + every.status === 'fulfilled' ? every.value.issues.map(({ document }) => document) : [] + if (!titled.length && half.status === 'fulfilled' && half.value) { + titled = half.value.issues + .map(({ document, summary }) => ({ document, held: terms.held(summary) })) + .filter(({ held }) => held >= terms.required) + .sort((left, right) => right.held - left.held) + .map(({ document }) => document) + } if (!titled.length) return exactPage const leading = new Set(titled.map((document) => document.id)) return { diff --git a/apps/sim/lib/sim-search/live/text-match.ts b/apps/sim/lib/sim-search/live/text-match.ts index d956efd0fac..9f3f1bfa237 100644 --- a/apps/sim/lib/sim-search/live/text-match.ts +++ b/apps/sim/lib/sim-search/live/text-match.ts @@ -130,11 +130,17 @@ export function termCoverage(text: string, terms: readonly string[]): number { * Query and text words are compared by their singular key, so a word and its regular plural count * as one term; a term with punctuation inside it is matched as written. */ -export function stemmedTerms(query: string): { count: number; held: (text: string) => number } { +export function stemmedTerms(query: string): { + count: number + /** At least half the terms, the minimum a partial match must hold. */ + required: number + held: (text: string) => number +} { const terms = distinctTerms(query, singularKey) const keys = terms.map(singularKey) return { count: terms.length, + required: Math.ceil(terms.length / 2), held: (text) => { const words = new Set(textWords(text).map(singularKey)) return terms.filter( @@ -144,9 +150,11 @@ export function stemmedTerms(query: string): { count: number; held: (text: strin } } -/** A test for whether a text holds at least half the query's distinct terms, as `stemmedTerms` counts them. */ +/** + * A test for whether a text holds at least half the query's distinct terms, as `stemmedTerms` + * counts them. + */ export function holdsHalfTheTerms(query: string): (text: string) => boolean { - const { count, held } = stemmedTerms(query) - const required = Math.ceil(count / 2) + const { required, held } = stemmedTerms(query) return (text) => held(text) >= required }