trimPrompt(item.content))
const prompt = [
- `Given the following contents from a SERP search for the query
${query}, generate a list of learnings from the contents. Return a maximum of ${numLearnings} learnings, but feel free to return less if the contents are clear. Make sure each learning is unique and not similar to each other. The learnings should be concise and to the point, as detailed and information dense as possible. Make sure to include any entities like people, places, companies, products, things, etc in the learnings, as well as any exact metrics, numbers, or dates. The learnings will be used to research the topic further.`,
+ `Given the following contents from a SERP search for the query
${query}, extract key learnings from the contents. For each learning, include the source URL. Return a maximum of ${numLearnings} learnings, but feel free to return less if the contents are clear. Make sure each learning is unique and not similar to each other. The learnings should be as detailed and information dense as possible. Include any entities like people, places, companies, products, things, etc in the learnings, as well as any exact metrics, numbers, or dates. Also generate up to ${numFollowUpQuestions} follow-up questions that could help explore this topic further.`,
`
${contents
- .map((content) => `\n${content}\n`)
+ .map(
+ (content, index) =>
+ `\n${content}\n`,
+ )
.join('\n')}`,
`You MUST respond in JSON matching this JSON schema: ${jsonSchema}`,
languagePrompt(language),
@@ -186,15 +209,18 @@ export function writeFinalReport({
}: WriteFinalReportParams) {
const learningsString = trimPrompt(
learnings
- .map((learning) => `
\n${learning}\n`)
+ .map(
+ (learning) =>
+ `
\n${learning.learning}\n`,
+ )
.join('\n'),
)
const _prompt = [
- `Given the following prompt from the user, write a final report on the topic using the learnings from research. Make it as as detailed as possible, aim for 3 or more pages, include ALL the learnings from research:`,
+ `Given the following prompt from the user, write a final report on the topic using the learnings from research. Make it as detailed as possible, aim for 3 or more pages, include ALL the key insights from research.`,
`
${prompt}`,
`Here are all the learnings from previous research:`,
`
\n${learningsString}\n`,
- `Write the report using Markdown.`,
+ `Write the report using Markdown. When citing information, use numbered citations with superscript numbers in square brackets (e.g., [1], [2], [3]). Each citation should correspond to the index of the source in your learnings list. DO NOT include the actual URLs in the report text - only use the citation numbers.`,
languagePrompt(language),
`## Deep Research Report`,
].join('\n\n')
@@ -219,8 +245,7 @@ export async function deepResearch({
maxDepth,
languageCode,
searchLanguageCode,
- learnings = [],
- visitedUrls = [],
+ learnings,
onProgress,
currentDepth,
nodeId = '0',
@@ -234,9 +259,7 @@ export async function deepResearch({
/** The language of SERP query */
searchLanguageCode?: Locale
/** Accumulated learnings from all nodes visited so far */
- learnings?: string[]
- /** Accumulated visited URLs from all nodes visited so far */
- visitedUrls?: string[]
+ learnings?: Array<{ url: string; learning: string }>
currentDepth: number
/** Current node ID. Used for recursive calls */
nodeId?: string
@@ -269,7 +292,7 @@ export async function deepResearch({
else {
const searchQueriesResult = generateSearchQueries({
query,
- learnings,
+ learnings: learnings?.map((item) => item.learning),
numQueries: breadth,
language,
searchLanguage,
@@ -362,8 +385,6 @@ export async function deepResearch({
`[DeepResearch] Searched "${searchQuery.query}", found ${results.length} contents`,
)
- // Collect URLs from this search
- const newUrls = results.map((item) => item.url).filter(Boolean)
onProgress({
type: 'search_complete',
results,
@@ -419,11 +440,17 @@ export async function deepResearch({
`Processed search result for ${searchQuery.query}`,
searchResult,
)
+ // Assign URL titles to learnings
+ searchResult.learnings = searchResult.learnings?.map((learning) => {
+ return {
+ ...learning,
+ title: results.find((r) => r.url === learning.url)?.title,
+ }
+ })
const allLearnings = [
- ...learnings,
+ ...(learnings ?? []),
...(searchResult.learnings ?? []),
]
- const allUrls = [...visitedUrls, ...newUrls]
const nextDepth = currentDepth + 1
onProgress({
@@ -458,7 +485,6 @@ export async function deepResearch({
breadth: nextBreadth,
maxDepth,
learnings: allLearnings,
- visitedUrls: allUrls,
onProgress,
currentDepth: nextDepth,
nodeId: searchQuery.nodeId,
@@ -473,7 +499,6 @@ export async function deepResearch({
} else {
return {
learnings: allLearnings,
- visitedUrls: allUrls,
}
}
} catch (e: any) {
@@ -488,26 +513,33 @@ export async function deepResearch({
})
return {
learnings: [],
- visitedUrls: [],
}
}
}),
),
)
// Conclude results
- const _learnings = [...new Set(results.flatMap((r) => r.learnings))]
- const _visitedUrls = [...new Set(results.flatMap((r) => r.visitedUrls))]
+ // Deduplicate
+ const urlMap = new Map
()
+ const finalLearnings: ProcessedSearchResult['learnings'] = []
+
+ for (const result of results) {
+ for (const learning of result.learnings) {
+ if (!urlMap.has(learning.url)) {
+ urlMap.set(learning.url, true)
+ finalLearnings.push(learning)
+ }
+ }
+ }
// Complete should only be called once
if (nodeId === '0') {
onProgress({
type: 'complete',
- learnings: _learnings,
- visitedUrls: _visitedUrls,
+ learnings: finalLearnings,
})
}
return {
- learnings: _learnings,
- visitedUrls: _visitedUrls,
+ learnings: finalLearnings,
}
} catch (error: any) {
console.error(error)
@@ -518,7 +550,6 @@ export async function deepResearch({
})
return {
learnings: [],
- visitedUrls: [],
}
}
}