improvement(search): fall back to Jira summaries holding half the terms of a longer query

apps/sim/lib/sim-search/live/atlassian.test.ts

@@ -334,16 +334,32 @@ describe('Confluence plain search ranking', () => {
 
 describe('Jira plain search ranking', () => {
   const search = '/ex/jira/cloud/rest/api/3/search/jql'
-  /** Answers the all-terms query with `exact` and the summary query with `titled`, or its failure. */
-  function jira(exact: { ids: string[]; more: boolean }, titled: string[] | (() => never)) {
-    const issues = (ids: string[]) => ids.map((key) => ({ key, fields: { summary: key } }))
-    const json = vi.fn(async (path: string, options?: { body?: { jql?: string } }) => {
-      if (path === '/oauth/token/accessible-resources') return [SITE]
-      if (path !== search) throw new Error(`Unexpected request: ${path}`)
-      if (options?.body?.jql?.includes('summary ~'))
-        return { issues: issues(typeof titled === 'function' ? titled() : titled) }
-      return { issues: issues(exact.ids), ...(exact.more ? { nextPageToken: 'page-2' } : {}) }
-    })
+  type Issue = string | { key: string; summary: string }
+  /**
+   * Answers the all-terms query with `exact`, the all-terms summary query with `titled`, and the
+   * half-the-terms summary query with `half`, or their failures.
+   */
+  function jira(
+    exact: { ids: string[]; more: boolean },
+    titled: Issue[] | (() => never),
+    half: Issue[] | (() => never) = []
+  ) {
+    const toRow = (row: Issue) => (typeof row === 'string' ? { key: row, summary: row } : row)
+    const issues = (rows: Issue[]) =>
+      rows.map(toRow).map(({ key, summary }) => ({ key, fields: { summary } }))
+    const json = vi.fn(
+      async (path: string, options?: { body?: { jql?: string; maxResults?: number } }) => {
+        if (path === '/oauth/token/accessible-resources') return [SITE]
+        if (path !== search) throw new Error(`Unexpected request: ${path}`)
+        const jql = options?.body?.jql ?? ''
+        if (jql.includes('summary ~')) {
+          const rows = jql.includes(' OR ') ? half : titled
+          const picked = typeof rows === 'function' ? rows() : rows
+          return { issues: issues(picked).slice(0, options?.body?.maxResults) }
+        }
+        return { issues: issues(exact.ids), ...(exact.more ? { nextPageToken: 'page-2' } : {}) }
+      }
+    )
     return { json, text: vi.fn(), bytes: vi.fn() } satisfies NativeClient
   }
   const query = { query: 'merged results pipelines', limit: 3, scopes: [] }
@@ -411,4 +427,66 @@ describe('Jira plain search ranking', () => {
       expect(api.json).toHaveBeenCalledTimes(2)
     }
   })
+
+  it('leads with issues whose summary holds at least half the terms when none holds them all', async () => {
+    const api = jira(
+      overflowing,
+      [],
+      [
+        { key: 'LOOSE-1', summary: 'Pipelines dashboard' },
+        { key: 'HALF-1', summary: 'Merged results broke the dashboard' },
+      ]
+    )
+    const page = await searchAtlassian(api, 'jira', query)
+    expect(page.documents.map((document) => document.id)).toEqual(['HALF-1', 'NEW-3', 'NEW-2'])
+    expect(page.nextCursor).toBeUndefined()
+  })
+
+  it('keeps a two-term query to summaries holding every term', async () => {
+    const api = jira(overflowing, [], [{ key: 'HALF-1', summary: 'Merged pipelines' }])
+    const page = await searchAtlassian(api, 'jira', { ...query, query: 'merged pipelines' })
+    expect(page.documents.map((document) => document.id)).toEqual(['NEW-3', 'NEW-2', 'NEW-1'])
+    expect(api.json).toHaveBeenCalledTimes(3)
+  })
+
+  it('keeps the all-terms page and its cursor when the half-the-terms summary search fails', async () => {
+    const api = jira(overflowing, [], () => {
+      throw new NativeSearchError('rate_limited', 'Rate limited.', 30)
+    })
+    const page = await searchAtlassian(api, 'jira', query)
+    expect(page.documents.map((document) => document.id)).toEqual(['NEW-3', 'NEW-2', 'NEW-1'])
+    expect(page.nextCursor).toBe('page-2')
+  })
+
+  it('counts a word and its plural as one term before allowing the half-the-terms fallback', async () => {
+    const api = jira(overflowing, [], [{ key: 'ONE-1', summary: 'Rotation schedule' }])
+    const page = await searchAtlassian(api, 'jira', { ...query, query: 'token tokens rotation' })
+    expect(page.documents.map((document) => document.id)).toEqual(['NEW-3', 'NEW-2', 'NEW-1'])
+  })
+
+  it('orders half-the-terms summaries by how many terms they hold', async () => {
+    const api = jira(
+      overflowing,
+      [],
+      [
+        { key: 'TWO-1', summary: 'Merged results dashboard' },
+        { key: 'THREE-1', summary: 'Merged results in release pipelines' },
+      ]
+    )
+    const page = await searchAtlassian(api, 'jira', {
+      ...query,
+      query: 'merged results release pipelines',
+    })
+    expect(page.documents.map((document) => document.id)).toEqual(['THREE-1', 'TWO-1', 'NEW-3'])
+  })
+
+  it('finds half-the-terms summaries beyond the first page of pair matches', async () => {
+    const pairs = Array.from({ length: 10 }, (_, index) => ({
+      key: `PAIR-${index}`,
+      summary: 'Pipelines dashboard',
+    }))
+    const api = jira(overflowing, [], [...pairs, { key: 'HALF-9', summary: 'Merged pipelines' }])
+    const page = await searchAtlassian(api, 'jira', query)
+    expect(page.documents.map((document) => document.id)[0]).toBe('HALF-9')
+  })
 })

apps/sim/lib/sim-search/live/atlassian.ts

@@ -12,8 +12,8 @@ import { providerText } from '@/lib/sim-search/live/text'
 import {
   confluenceTextClause,
   holdsHalfTheTerms,
-  jiraSummaryClause,
   jiraTextClause,
+  stemmedTerms,
 } from '@/lib/sim-search/live/text-match'
 import type {
   NativeClient,
@@ -117,6 +117,13 @@ async function sites(client: NativeClient) {
   return array(await client.json('/oauth/token/accessible-resources', { memo: true }))
 }
 
+/**
+ * Pair matches fetched for the half-the-terms summary fallback. Jira returns them newest first,
+ * not by relevance, and the page is then filtered to summaries holding enough terms, so it starts
+ * from a wider pool than the caller's limit or the any-term fallback's.
+ */
+const HALF_TERM_CANDIDATES = 50
+
 /**
  * One Atlassian site's page; `ranking` marks a page whose order or completeness differs from the
  * all-terms search.
@@ -176,55 +183,82 @@ export async function searchAtlassian(
       const cloudId = string(site.id)
       const origin = string(site.url).replace(/\/$/, '')
       if (provider === 'jira') {
-        const searchIssues = async (jql: string, cursor?: string): Promise<SitePage> => {
+        /** One JQL page, each issue with its summary and raw fields. */
+        const searchIssues = async (
+          jql: string,
+          { cursor, maxResults = input.limit }: { cursor?: string; maxResults?: number } = {}
+        ) => {
           const data = object(
             await client.json(`/ex/jira/${segment(cloudId)}/rest/api/3/search/jql`, {
               body: {
                 jql: scopeAtlassianQuery(jql, scope, order),
-                maxResults: input.limit,
+                maxResults,
                 fields: ['summary', 'description', 'updated', 'creator', 'status'],
                 ...(cursor ? { nextPageToken: cursor } : {}),
               },
               signal: input.signal,
             })
           )
           return {
-            documents: array(data.issues)
-              .filter((row) => {
-                const fields = object(row.fields)
-                return (
-                  !relevant ||
-                  relevant(`${string(fields.summary)}\n${textContent(fields.description)}`)
-                )
-              })
-              .map((row) => issue(row, cloudId, origin)),
-            excluded: false,
+            issues: array(data.issues).map((row) => {
+              const fields = object(row.fields)
+              return {
+                document: issue(row, cloudId, origin),
+                summary: string(fields.summary),
+                fields,
+              }
+            }),
             next: string(data.nextPageToken) || undefined,
           }
         }
         const exact = await searchIssues(
           input.native?.query || (text ? jiraTextClause(text, input.textMatch) : ''),
-          input.native?.cursor && single ? input.native.cursor : undefined
+          { cursor: input.native?.cursor && single ? input.native.cursor : undefined }
         )
+        const exactPage: SitePage = {
+          documents: exact.issues
+            .filter(
+              ({ summary, fields }) =>
+                !relevant || relevant(`${summary}\n${textContent(fields.description)}`)
+            )
+            .map(({ document }) => document),
+          excluded: false,
+          next: exact.next,
+        }
         // JQL has no relevance order, so an all-terms match larger than a page comes back newest
-        // first; issues whose summary holds every term lead it instead.
-        if (!text || input.native || order || input.textMatch === 'any' || !exact.next) return exact
-        try {
-          const titled = await searchIssues(jiraSummaryClause(text))
-          if (!titled.documents.length) return exact
-          const leading = new Set(titled.documents.map((document) => document.id))
-          return {
-            documents: [
-              ...titled.documents,
-              ...exact.documents.filter((document) => !leading.has(document.id)),
-            ].slice(0, input.limit),
-            excluded: false,
-            ranking: 'relevance',
-          }
-        } catch {
-          // A cancelled search stops here; any other failure keeps the all-terms page.
-          input.signal?.throwIfAborted()
-          return exact
+        // first; issues whose summary holds every term, or else at least half of three or more
+        // terms (most terms first), lead it instead.
+        if (!text || input.native || order || input.textMatch === 'any' || !exact.next)
+          return exactPage
+        const terms = stemmedTerms(text)
+        const [every, half] = await Promise.allSettled([
+          searchIssues(jiraTextClause(text, 'all', 'summary')),
+          terms.count >= 3
+            ? searchIssues(jiraTextClause(text, 'any', 'summary'), {
+                maxResults: HALF_TERM_CANDIDATES,
+              })
+            : undefined,
+        ])
+        // A cancelled search stops here; any other failure keeps the all-terms page.
+        input.signal?.throwIfAborted()
+        let titled =
+          every.status === 'fulfilled' ? every.value.issues.map(({ document }) => document) : []
+        if (!titled.length && half.status === 'fulfilled' && half.value) {
+          titled = half.value.issues
+            .map(({ document, summary }) => ({ document, held: terms.held(summary) }))
+            .filter(({ held }) => held >= terms.required)
+            .sort((left, right) => right.held - left.held)
+            .map(({ document }) => document)
+        }
+        if (!titled.length) return exactPage
+        const leading = new Set(titled.map((document) => document.id))
+        return {
+          documents: [
+            ...titled,
+            ...exactPage.documents.filter((document) => !leading.has(document.id)),
+          ].slice(0, input.limit),
+          excluded: false,
+          ranking: 'relevance',
         }
       }
       const searchPages = async (clause: string, cursor?: string) => {
@@ -289,7 +323,7 @@ export async function searchAtlassian(
         : '') +
       (pages.some((result) => result.ranking === 'relevance')
         ? provider === 'jira'
-          ? ' More issues contained every word than fit on one page, so issues whose summary holds every word come first; refine the query, or page with a native JQL query.'
+          ? ' More issues contained every word than fit on one page, so issues whose summary holds every word, or else most of them, come first; refine the query, or page with a native JQL query.'
           : ' More pages contained every word than fit on one page, so this page is ordered by Confluence relevance search; refine the query, or page with a native CQL query.'
         : '') +
       (pages.some((result) => result.ranking === 'unavailable')

apps/sim/lib/sim-search/live/text-match.ts

@@ -89,14 +89,19 @@ const escapeSearchPhrase = (value: string) =>
   JSON.stringify(value.replace(/[+\-&|!(){}[\]^"~*?:\\/]/g, '\\$&'))
 
 /**
- * The `text ~` clause for a plain query in Jira JQL. `any` ORs every pair of the query's longest
- * terms, so a match contains at least two of them somewhere in the issue, or either term of a
- * two-term query. Jira stems text search, so a word and its plural share one term.
+ * The clause for a plain query in Jira JQL, over all text fields or the summary alone. `any` ORs
+ * every pair of the query's longest terms, so a match contains at least two of them in that field,
+ * or either term of a two-term query. Jira stems text search, so a word and its plural share one
+ * term.
  */
-export function jiraTextClause(text: string, match: TextMatch = 'all'): string {
+export function jiraTextClause(
+  text: string,
+  match: TextMatch = 'all',
+  field: 'text' | 'summary' = 'text'
+): string {
   const terms = match === 'any' ? distinctTerms(text, singularKey) : []
-  if (terms.length < 2) return `text ~ ${escapeSearchPhrase(text)}`
-  const clause = (term: string) => `text ~ ${escapeSearchPhrase(term)}`
+  if (terms.length < 2) return `${field} ~ ${escapeSearchPhrase(text)}`
+  const clause = (term: string) => `${field} ~ ${escapeSearchPhrase(term)}`
   const groups =
     terms.length === 2
       ? terms.map(clause)
@@ -106,14 +111,6 @@ export function jiraTextClause(text: string, match: TextMatch = 'all'): string {
   return `(${groups.join(' OR ')})`
 }
 
-/**
- * The `summary ~` clause for a plain query in Jira JQL: issues whose summary holds every word.
- * JQL has no relevance order, so this stands in for a title boost.
- */
-export function jiraSummaryClause(text: string): string {
-  return `summary ~ ${escapeSearchPhrase(text)}`
-}
-
 /**
  * The text clause for a plain query in Confluence CQL. `any` uses `siteSearch`, Confluence's own
  * relevance search, which ranks a page with the query's rarer words above the many pages sharing
@@ -129,19 +126,35 @@ export function termCoverage(text: string, terms: readonly string[]): number {
 }
 
 /**
- * A test for whether a text holds at least half the query's distinct terms. Query and text words
- * are compared by their singular key, so a word and its regular plural count as one term as a
- * stemmed search matches them; a term with punctuation inside it is matched as written.
+ * The query's distinct terms as a stemmed search counts them, and how many of them a text holds.
+ * Query and text words are compared by their singular key, so a word and its regular plural count
+ * as one term; a term with punctuation inside it is matched as written.
  */
-export function holdsHalfTheTerms(query: string): (text: string) => boolean {
+export function stemmedTerms(query: string): {
+  count: number
+  /** At least half the terms, the minimum a partial match must hold. */
+  required: number
+  held: (text: string) => number
+} {
   const terms = distinctTerms(query, singularKey)
   const keys = terms.map(singularKey)
-  const required = Math.ceil(terms.length / 2)
-  return (text) => {
-    const words = new Set(textWords(text).map(singularKey))
-    const present = terms.filter(
-      (term, index) => words.has(keys[index]) || findTermMatches(text, [term]).length > 0
-    )
-    return present.length >= required
+  return {
+    count: terms.length,
+    required: Math.ceil(terms.length / 2),
+    held: (text) => {
+      const words = new Set(textWords(text).map(singularKey))
+      return terms.filter(
+        (term, index) => words.has(keys[index]) || findTermMatches(text, [term]).length > 0
+      ).length
+    },
   }
 }
+
+/**
+ * A test for whether a text holds at least half the query's distinct terms, as `stemmedTerms`
+ * counts them.
+ */
+export function holdsHalfTheTerms(query: string): (text: string) => boolean {
+  const { required, held } = stemmedTerms(query)
+  return (text) => held(text) >= required
+}