style: prettier format
This commit is contained in:
11
README.md
11
README.md
@ -2,12 +2,9 @@
|
|||||||
|
|
||||||
This is a web UI for https://github.com/dzhng/deep-research. It supports streaming AI responses, and viasualization of the research process using a tree structure.
|
This is a web UI for https://github.com/dzhng/deep-research. It supports streaming AI responses, and viasualization of the research process using a tree structure.
|
||||||
|
|
||||||
Note: The project is currently WIP, expect bugs. README will be updated once the project is usable.
|
> Note: The project is currently WIP, expect bugs.
|
||||||
|
|
||||||
Rough preview of the UI:
|
|
||||||
|
|
||||||
<img width="1087" alt="image" src="https://github.com/user-attachments/assets/4bb5b722-0300-4d4f-bb01-fc1ed2404442" />
|
|
||||||
|
|
||||||
|
<video src="https://github.com/user-attachments/assets/c3738551-b258-47c6-90a8-fd097e5165c8"></video>
|
||||||
|
|
||||||
## Setup
|
## Setup
|
||||||
|
|
||||||
@ -80,3 +77,7 @@ bun run preview
|
|||||||
```
|
```
|
||||||
|
|
||||||
Check out the [deployment documentation](https://nuxt.com/docs/getting-started/deployment) for more information.
|
Check out the [deployment documentation](https://nuxt.com/docs/getting-started/deployment) for more information.
|
||||||
|
|
||||||
|
## License
|
||||||
|
|
||||||
|
MIT
|
||||||
|
21
app.vue
21
app.vue
@ -18,9 +18,15 @@
|
|||||||
content:
|
content:
|
||||||
'SiliconCloud Stats 是一个用于分析 SiliconCloud 平台使用情况的工具。通过输入 Cookie,可以拉取 SiliconCloud 控制台 API 来实现各种分析功能,如 token 用量分析等。',
|
'SiliconCloud Stats 是一个用于分析 SiliconCloud 平台使用情况的工具。通过输入 Cookie,可以拉取 SiliconCloud 控制台 API 来实现各种分析功能,如 token 用量分析等。',
|
||||||
},
|
},
|
||||||
{ name: 'keywords', content: 'SiliconCloud, 数据分析, token 用量, API 分析, 控制台工具' },
|
{
|
||||||
|
name: 'keywords',
|
||||||
|
content: 'SiliconCloud, 数据分析, token 用量, API 分析, 控制台工具',
|
||||||
|
},
|
||||||
// Open Graph tags
|
// Open Graph tags
|
||||||
{ property: 'og:title', content: 'SiliconCloud Stats - SiliconCloud 平台使用情况分析工具' },
|
{
|
||||||
|
property: 'og:title',
|
||||||
|
content: 'SiliconCloud Stats - SiliconCloud 平台使用情况分析工具',
|
||||||
|
},
|
||||||
{
|
{
|
||||||
property: 'og:description',
|
property: 'og:description',
|
||||||
content:
|
content:
|
||||||
@ -30,8 +36,15 @@
|
|||||||
{ property: 'og:image', content: '/images/readme-showcase-total.webp' },
|
{ property: 'og:image', content: '/images/readme-showcase-total.webp' },
|
||||||
// Twitter Card tags
|
// Twitter Card tags
|
||||||
{ name: 'twitter:card', content: 'summary_large_image' },
|
{ name: 'twitter:card', content: 'summary_large_image' },
|
||||||
{ name: 'twitter:title', content: 'SiliconCloud Stats - SiliconCloud 平台使用情况分析工具' },
|
{
|
||||||
{ name: 'twitter:description', content: 'SiliconCloud Stats 是一个用于分析 SiliconCloud 平台使用情况的工具。' },
|
name: 'twitter:title',
|
||||||
|
content: 'SiliconCloud Stats - SiliconCloud 平台使用情况分析工具',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'twitter:description',
|
||||||
|
content:
|
||||||
|
'SiliconCloud Stats 是一个用于分析 SiliconCloud 平台使用情况的工具。',
|
||||||
|
},
|
||||||
{ name: 'twitter:image', content: '/images/readme-showcase-total.webp' },
|
{ name: 'twitter:image', content: '/images/readme-showcase-total.webp' },
|
||||||
],
|
],
|
||||||
// script: [
|
// script: [
|
||||||
|
@ -19,6 +19,10 @@
|
|||||||
|
|
||||||
<template>
|
<template>
|
||||||
<div>
|
<div>
|
||||||
<UButton :icon="preference === 'dark' ? 'i-lucide-sun' : 'i-lucide-moon'" color="primary" @click="toggleColorMode" />
|
<UButton
|
||||||
|
:icon="preference === 'dark' ? 'i-lucide-sun' : 'i-lucide-moon'"
|
||||||
|
color="primary"
|
||||||
|
@click="toggleColorMode"
|
||||||
|
/>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
|
@ -1,5 +1,10 @@
|
|||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { deepResearch, type PartialSearchResult, type ResearchResult, type ResearchStep } from '~/lib/deep-research'
|
import {
|
||||||
|
deepResearch,
|
||||||
|
type PartialSearchResult,
|
||||||
|
type ResearchResult,
|
||||||
|
type ResearchStep,
|
||||||
|
} from '~/lib/deep-research'
|
||||||
import type { TreeNode } from './Tree.vue'
|
import type { TreeNode } from './Tree.vue'
|
||||||
|
|
||||||
const emit = defineEmits<{
|
const emit = defineEmits<{
|
||||||
@ -44,7 +49,10 @@
|
|||||||
tree.value.children.push(node)
|
tree.value.children.push(node)
|
||||||
} else {
|
} else {
|
||||||
// 找到父节点并添加
|
// 找到父节点并添加
|
||||||
const parentNode = findNode(tree.value, getParentNodeId(step.nodeId))
|
const parentNode = findNode(
|
||||||
|
tree.value,
|
||||||
|
getParentNodeId(step.nodeId),
|
||||||
|
)
|
||||||
if (parentNode) {
|
if (parentNode) {
|
||||||
parentNode.children.push(node)
|
parentNode.children.push(node)
|
||||||
}
|
}
|
||||||
@ -160,7 +168,8 @@
|
|||||||
<template #header>
|
<template #header>
|
||||||
<h2 class="font-bold">3. Web Browsing</h2>
|
<h2 class="font-bold">3. Web Browsing</h2>
|
||||||
<p class="text-sm text-gray-500">
|
<p class="text-sm text-gray-500">
|
||||||
The AI will then search the web based on our research goal, and iterate until the depth is reached.
|
The AI will then search the web based on our research goal, and iterate
|
||||||
|
until the depth is reached.
|
||||||
<br />
|
<br />
|
||||||
Click a child node to view details.
|
Click a child node to view details.
|
||||||
</p>
|
</p>
|
||||||
@ -174,7 +183,9 @@
|
|||||||
<h2 class="text-xl font-bold mt-2">{{ selectedNode.label }}</h2>
|
<h2 class="text-xl font-bold mt-2">{{ selectedNode.label }}</h2>
|
||||||
|
|
||||||
<!-- Root node has no additional information -->
|
<!-- Root node has no additional information -->
|
||||||
<p v-if="selectedNode.id === '0'"> This is the beginning of your deep research journey! </p>
|
<p v-if="selectedNode.id === '0'">
|
||||||
|
This is the beginning of your deep research journey!
|
||||||
|
</p>
|
||||||
<template v-else>
|
<template v-else>
|
||||||
<h3 class="text-lg font-semibold mt-2">Research Goal:</h3>
|
<h3 class="text-lg font-semibold mt-2">Research Goal:</h3>
|
||||||
<p>{{ selectedNode.researchGoal }}</p>
|
<p>{{ selectedNode.researchGoal }}</p>
|
||||||
@ -188,7 +199,11 @@
|
|||||||
|
|
||||||
<h3 class="text-lg font-semibold mt-2">Learnings:</h3>
|
<h3 class="text-lg font-semibold mt-2">Learnings:</h3>
|
||||||
<ul class="list-disc list-inside">
|
<ul class="list-disc list-inside">
|
||||||
<li v-for="(learning, index) in selectedNode.learnings" :key="index">{{ learning }}</li>
|
<li
|
||||||
|
v-for="(learning, index) in selectedNode.learnings"
|
||||||
|
:key="index"
|
||||||
|
>{{ learning }}</li
|
||||||
|
>
|
||||||
</ul>
|
</ul>
|
||||||
</template>
|
</template>
|
||||||
</div>
|
</div>
|
||||||
|
@ -74,7 +74,10 @@
|
|||||||
<UCard>
|
<UCard>
|
||||||
<template #header>
|
<template #header>
|
||||||
<h2 class="font-bold">2. Model Feedback</h2>
|
<h2 class="font-bold">2. Model Feedback</h2>
|
||||||
<p class="text-sm text-gray-500"> The AI will ask you some follow up questions to help you clarify the research direction. </p>
|
<p class="text-sm text-gray-500">
|
||||||
|
The AI will ask you some follow up questions to help you clarify the
|
||||||
|
research direction.
|
||||||
|
</p>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<div class="flex flex-col gap-2">
|
<div class="flex flex-col gap-2">
|
||||||
@ -82,7 +85,11 @@
|
|||||||
<div v-if="!feedback.length && !error">Waiting for model feedback...</div>
|
<div v-if="!feedback.length && !error">Waiting for model feedback...</div>
|
||||||
<template v-else>
|
<template v-else>
|
||||||
<div v-if="error" class="text-red-500">{{ error }}</div>
|
<div v-if="error" class="text-red-500">{{ error }}</div>
|
||||||
<div v-for="(feedback, index) in feedback" class="flex flex-col gap-2" :key="index">
|
<div
|
||||||
|
v-for="(feedback, index) in feedback"
|
||||||
|
class="flex flex-col gap-2"
|
||||||
|
:key="index"
|
||||||
|
>
|
||||||
Assistant: {{ feedback.assistantQuestion }}
|
Assistant: {{ feedback.assistantQuestion }}
|
||||||
<UInput v-model="feedback.userAnswer" />
|
<UInput v-model="feedback.userAnswer" />
|
||||||
</div>
|
</div>
|
||||||
|
@ -21,7 +21,9 @@
|
|||||||
numQuestions: 3,
|
numQuestions: 3,
|
||||||
})
|
})
|
||||||
|
|
||||||
const isSubmitButtonDisabled = computed(() => !form.query || !form.breadth || !form.depth || !form.numQuestions)
|
const isSubmitButtonDisabled = computed(
|
||||||
|
() => !form.query || !form.breadth || !form.depth || !form.numQuestions,
|
||||||
|
)
|
||||||
|
|
||||||
function handleSubmit() {
|
function handleSubmit() {
|
||||||
emit('submit', {
|
emit('submit', {
|
||||||
@ -41,29 +43,63 @@
|
|||||||
</template>
|
</template>
|
||||||
<div class="flex flex-col gap-2">
|
<div class="flex flex-col gap-2">
|
||||||
<UFormField label="Research Topic" required>
|
<UFormField label="Research Topic" required>
|
||||||
<UTextarea class="w-full" v-model="form.query" :rows="3" placeholder="Enter whatever you want to research..." required />
|
<UTextarea
|
||||||
|
class="w-full"
|
||||||
|
v-model="form.query"
|
||||||
|
:rows="3"
|
||||||
|
placeholder="Enter whatever you want to research..."
|
||||||
|
required
|
||||||
|
/>
|
||||||
</UFormField>
|
</UFormField>
|
||||||
|
|
||||||
<div class="grid grid-cols-1 sm:grid-cols-3 gap-4">
|
<div class="grid grid-cols-1 sm:grid-cols-3 gap-4">
|
||||||
<UFormField label="Number of Questions" required>
|
<UFormField label="Number of Questions" required>
|
||||||
<template #help> Number of questions for you to clarify. </template>
|
<template #help> Number of questions for you to clarify. </template>
|
||||||
<UInput v-model="form.numQuestions" class="w-full" type="number" :min="1" :max="5" :step="1" />
|
<UInput
|
||||||
|
v-model="form.numQuestions"
|
||||||
|
class="w-full"
|
||||||
|
type="number"
|
||||||
|
:min="1"
|
||||||
|
:max="5"
|
||||||
|
:step="1"
|
||||||
|
/>
|
||||||
</UFormField>
|
</UFormField>
|
||||||
|
|
||||||
<UFormField label="Depth" required>
|
<UFormField label="Depth" required>
|
||||||
<template #help> How deep you want to dig. </template>
|
<template #help> How deep you want to dig. </template>
|
||||||
<UInput v-model="form.depth" class="w-full" type="number" :min="1" :max="5" :step="1" />
|
<UInput
|
||||||
|
v-model="form.depth"
|
||||||
|
class="w-full"
|
||||||
|
type="number"
|
||||||
|
:min="1"
|
||||||
|
:max="5"
|
||||||
|
:step="1"
|
||||||
|
/>
|
||||||
</UFormField>
|
</UFormField>
|
||||||
|
|
||||||
<UFormField label="Breadth" required>
|
<UFormField label="Breadth" required>
|
||||||
<template #help> Number of searches in each depth. </template>
|
<template #help> Number of searches in each depth. </template>
|
||||||
<UInput v-model="form.breadth" class="w-full" type="number" :min="1" :max="5" :step="1" />
|
<UInput
|
||||||
|
v-model="form.breadth"
|
||||||
|
class="w-full"
|
||||||
|
type="number"
|
||||||
|
:min="1"
|
||||||
|
:max="5"
|
||||||
|
:step="1"
|
||||||
|
/>
|
||||||
</UFormField>
|
</UFormField>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<template #footer>
|
<template #footer>
|
||||||
<UButton type="submit" color="primary" :loading="isLoadingFeedback" :disabled="isSubmitButtonDisabled" block @click="handleSubmit">
|
<UButton
|
||||||
|
type="submit"
|
||||||
|
color="primary"
|
||||||
|
:loading="isLoadingFeedback"
|
||||||
|
:disabled="isSubmitButtonDisabled"
|
||||||
|
block
|
||||||
|
@click="handleSubmit"
|
||||||
|
>
|
||||||
{{ isLoadingFeedback ? 'Researching...' : 'Start Research' }}
|
{{ isLoadingFeedback ? 'Researching...' : 'Start Research' }}
|
||||||
</UButton>
|
</UButton>
|
||||||
</template>
|
</template>
|
||||||
|
@ -1,6 +1,9 @@
|
|||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { marked } from 'marked'
|
import { marked } from 'marked'
|
||||||
import { writeFinalReport, type WriteFinalReportParams } from '~/lib/deep-research'
|
import {
|
||||||
|
writeFinalReport,
|
||||||
|
type WriteFinalReportParams,
|
||||||
|
} from '~/lib/deep-research'
|
||||||
|
|
||||||
interface CustomReportParams extends WriteFinalReportParams {
|
interface CustomReportParams extends WriteFinalReportParams {
|
||||||
visitedUrls: string[]
|
visitedUrls: string[]
|
||||||
@ -10,8 +13,12 @@
|
|||||||
const loading = ref(false)
|
const loading = ref(false)
|
||||||
const loadingExportPdf = ref(false)
|
const loadingExportPdf = ref(false)
|
||||||
const reportContent = ref('')
|
const reportContent = ref('')
|
||||||
const reportHtml = computed(() => marked(reportContent.value, { gfm: true, silent: true }))
|
const reportHtml = computed(() =>
|
||||||
const isExportButtonDisabled = computed(() => !reportContent.value || loading.value || loadingExportPdf.value)
|
marked(reportContent.value, { gfm: true, silent: true }),
|
||||||
|
)
|
||||||
|
const isExportButtonDisabled = computed(
|
||||||
|
() => !reportContent.value || loading.value || loadingExportPdf.value,
|
||||||
|
)
|
||||||
|
|
||||||
async function generateReport(params: CustomReportParams) {
|
async function generateReport(params: CustomReportParams) {
|
||||||
loading.value = true
|
loading.value = true
|
||||||
@ -113,7 +120,9 @@
|
|||||||
/>
|
/>
|
||||||
<template v-else>
|
<template v-else>
|
||||||
<div v-if="error" class="text-red-500">{{ error }}</div>
|
<div v-if="error" class="text-red-500">{{ error }}</div>
|
||||||
<div v-else>{{ loading ? 'Generating report...' : 'Waiting for report..' }}.</div>
|
<div v-else
|
||||||
|
>{{ loading ? 'Generating report...' : 'Waiting for report..' }}.</div
|
||||||
|
>
|
||||||
</template>
|
</template>
|
||||||
</UCard>
|
</UCard>
|
||||||
</template>
|
</template>
|
||||||
|
@ -72,7 +72,12 @@
|
|||||||
</UButton>
|
</UButton>
|
||||||
<ol v-if="node.children.length > 0" class="space-y-2">
|
<ol v-if="node.children.length > 0" class="space-y-2">
|
||||||
<li v-for="node in node.children" :key="node.id">
|
<li v-for="node in node.children" :key="node.id">
|
||||||
<Tree class="ml-2" :node="node" :selected-node @select="emit('select', $event)" />
|
<Tree
|
||||||
|
class="ml-2"
|
||||||
|
:node="node"
|
||||||
|
:selected-node
|
||||||
|
@select="emit('select', $event)"
|
||||||
|
/>
|
||||||
</li>
|
</li>
|
||||||
</ol>
|
</ol>
|
||||||
</div>
|
</div>
|
||||||
|
@ -1,25 +1,25 @@
|
|||||||
import { createOpenAI } from '@ai-sdk/openai';
|
import { createOpenAI } from '@ai-sdk/openai'
|
||||||
import { getEncoding } from 'js-tiktoken';
|
import { getEncoding } from 'js-tiktoken'
|
||||||
|
|
||||||
import { RecursiveCharacterTextSplitter } from './text-splitter';
|
import { RecursiveCharacterTextSplitter } from './text-splitter'
|
||||||
|
|
||||||
// Providers
|
// Providers
|
||||||
const openai = createOpenAI({
|
const openai = createOpenAI({
|
||||||
apiKey: import.meta.env.VITE_OPENAI_API_KEY!,
|
apiKey: import.meta.env.VITE_OPENAI_API_KEY!,
|
||||||
baseURL: import.meta.env.VITE_OPENAI_ENDPOINT || 'https://api.openai.com/v1',
|
baseURL: import.meta.env.VITE_OPENAI_ENDPOINT || 'https://api.openai.com/v1',
|
||||||
});
|
})
|
||||||
|
|
||||||
const customModel = import.meta.env.VITE_OPENAI_MODEL || 'o3-mini';
|
const customModel = import.meta.env.VITE_OPENAI_MODEL || 'o3-mini'
|
||||||
|
|
||||||
// Models
|
// Models
|
||||||
|
|
||||||
export const o3MiniModel = openai(customModel, {
|
export const o3MiniModel = openai(customModel, {
|
||||||
// reasoningEffort: customModel.startsWith('o') ? 'medium' : undefined,
|
// reasoningEffort: customModel.startsWith('o') ? 'medium' : undefined,
|
||||||
structuredOutputs: true,
|
structuredOutputs: true,
|
||||||
});
|
})
|
||||||
|
|
||||||
const MinChunkSize = 140;
|
const MinChunkSize = 140
|
||||||
const encoder = getEncoding('o200k_base');
|
const encoder = getEncoding('o200k_base')
|
||||||
|
|
||||||
// trim prompt to maximum context size
|
// trim prompt to maximum context size
|
||||||
export function trimPrompt(
|
export function trimPrompt(
|
||||||
@ -27,32 +27,32 @@ export function trimPrompt(
|
|||||||
contextSize = Number(import.meta.env.VITE_CONTEXT_SIZE) || 128_000,
|
contextSize = Number(import.meta.env.VITE_CONTEXT_SIZE) || 128_000,
|
||||||
) {
|
) {
|
||||||
if (!prompt) {
|
if (!prompt) {
|
||||||
return '';
|
return ''
|
||||||
}
|
}
|
||||||
|
|
||||||
const length = encoder.encode(prompt).length;
|
const length = encoder.encode(prompt).length
|
||||||
if (length <= contextSize) {
|
if (length <= contextSize) {
|
||||||
return prompt;
|
return prompt
|
||||||
}
|
}
|
||||||
|
|
||||||
const overflowTokens = length - contextSize;
|
const overflowTokens = length - contextSize
|
||||||
// on average it's 3 characters per token, so multiply by 3 to get a rough estimate of the number of characters
|
// on average it's 3 characters per token, so multiply by 3 to get a rough estimate of the number of characters
|
||||||
const chunkSize = prompt.length - overflowTokens * 3;
|
const chunkSize = prompt.length - overflowTokens * 3
|
||||||
if (chunkSize < MinChunkSize) {
|
if (chunkSize < MinChunkSize) {
|
||||||
return prompt.slice(0, MinChunkSize);
|
return prompt.slice(0, MinChunkSize)
|
||||||
}
|
}
|
||||||
|
|
||||||
const splitter = new RecursiveCharacterTextSplitter({
|
const splitter = new RecursiveCharacterTextSplitter({
|
||||||
chunkSize,
|
chunkSize,
|
||||||
chunkOverlap: 0,
|
chunkOverlap: 0,
|
||||||
});
|
})
|
||||||
const trimmedPrompt = splitter.splitText(prompt)[0] ?? '';
|
const trimmedPrompt = splitter.splitText(prompt)[0] ?? ''
|
||||||
|
|
||||||
// last catch, there's a chance that the trimmed prompt is same length as the original prompt, due to how tokens are split & innerworkings of the splitter, handle this case by just doing a hard cut
|
// last catch, there's a chance that the trimmed prompt is same length as the original prompt, due to how tokens are split & innerworkings of the splitter, handle this case by just doing a hard cut
|
||||||
if (trimmedPrompt.length === prompt.length) {
|
if (trimmedPrompt.length === prompt.length) {
|
||||||
return trimPrompt(prompt.slice(0, chunkSize), contextSize);
|
return trimPrompt(prompt.slice(0, chunkSize), contextSize)
|
||||||
}
|
}
|
||||||
|
|
||||||
// recursively trim until the prompt is within the context size
|
// recursively trim until the prompt is within the context size
|
||||||
return trimPrompt(trimmedPrompt, contextSize);
|
return trimPrompt(trimmedPrompt, contextSize)
|
||||||
}
|
}
|
||||||
|
@ -1,77 +1,80 @@
|
|||||||
import assert from 'node:assert';
|
import assert from 'node:assert'
|
||||||
import { describe, it, beforeEach } from 'node:test';
|
import { describe, it, beforeEach } from 'node:test'
|
||||||
import { RecursiveCharacterTextSplitter } from './text-splitter';
|
import { RecursiveCharacterTextSplitter } from './text-splitter'
|
||||||
|
|
||||||
describe('RecursiveCharacterTextSplitter', () => {
|
describe('RecursiveCharacterTextSplitter', () => {
|
||||||
let splitter: RecursiveCharacterTextSplitter;
|
let splitter: RecursiveCharacterTextSplitter
|
||||||
|
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
splitter = new RecursiveCharacterTextSplitter({
|
splitter = new RecursiveCharacterTextSplitter({
|
||||||
chunkSize: 50,
|
chunkSize: 50,
|
||||||
chunkOverlap: 10,
|
chunkOverlap: 10,
|
||||||
});
|
})
|
||||||
});
|
})
|
||||||
|
|
||||||
it('Should correctly split text by separators', () => {
|
it('Should correctly split text by separators', () => {
|
||||||
const text = 'Hello world, this is a test of the recursive text splitter.';
|
const text = 'Hello world, this is a test of the recursive text splitter.'
|
||||||
|
|
||||||
// Test with initial chunkSize
|
// Test with initial chunkSize
|
||||||
assert.deepEqual(
|
assert.deepEqual(splitter.splitText(text), [
|
||||||
splitter.splitText(text),
|
'Hello world',
|
||||||
['Hello world', 'this is a test of the recursive text splitter']
|
'this is a test of the recursive text splitter',
|
||||||
);
|
])
|
||||||
|
|
||||||
// Test with updated chunkSize
|
// Test with updated chunkSize
|
||||||
splitter.chunkSize = 100;
|
splitter.chunkSize = 100
|
||||||
assert.deepEqual(
|
assert.deepEqual(
|
||||||
splitter.splitText(
|
splitter.splitText(
|
||||||
'Hello world, this is a test of the recursive text splitter. If I have a period, it should split along the period.'
|
'Hello world, this is a test of the recursive text splitter. If I have a period, it should split along the period.',
|
||||||
),
|
),
|
||||||
[
|
[
|
||||||
'Hello world, this is a test of the recursive text splitter',
|
'Hello world, this is a test of the recursive text splitter',
|
||||||
'If I have a period, it should split along the period.',
|
'If I have a period, it should split along the period.',
|
||||||
]
|
],
|
||||||
);
|
)
|
||||||
|
|
||||||
// Test with another updated chunkSize
|
// Test with another updated chunkSize
|
||||||
splitter.chunkSize = 110;
|
splitter.chunkSize = 110
|
||||||
assert.deepEqual(
|
assert.deepEqual(
|
||||||
splitter.splitText(
|
splitter.splitText(
|
||||||
'Hello world, this is a test of the recursive text splitter. If I have a period, it should split along the period.\nOr, if there is a new line, it should prioritize splitting on new lines instead.'
|
'Hello world, this is a test of the recursive text splitter. If I have a period, it should split along the period.\nOr, if there is a new line, it should prioritize splitting on new lines instead.',
|
||||||
),
|
),
|
||||||
[
|
[
|
||||||
'Hello world, this is a test of the recursive text splitter',
|
'Hello world, this is a test of the recursive text splitter',
|
||||||
'If I have a period, it should split along the period.',
|
'If I have a period, it should split along the period.',
|
||||||
'Or, if there is a new line, it should prioritize splitting on new lines instead.',
|
'Or, if there is a new line, it should prioritize splitting on new lines instead.',
|
||||||
]
|
],
|
||||||
);
|
)
|
||||||
});
|
})
|
||||||
|
|
||||||
it('Should handle empty string', () => {
|
it('Should handle empty string', () => {
|
||||||
assert.deepEqual(splitter.splitText(''), []);
|
assert.deepEqual(splitter.splitText(''), [])
|
||||||
});
|
})
|
||||||
|
|
||||||
it('Should handle special characters and large texts', () => {
|
it('Should handle special characters and large texts', () => {
|
||||||
const largeText = 'A'.repeat(1000);
|
const largeText = 'A'.repeat(1000)
|
||||||
splitter.chunkSize = 200;
|
splitter.chunkSize = 200
|
||||||
assert.deepEqual(
|
assert.deepEqual(
|
||||||
splitter.splitText(largeText),
|
splitter.splitText(largeText),
|
||||||
Array(5).fill('A'.repeat(200))
|
Array(5).fill('A'.repeat(200)),
|
||||||
);
|
)
|
||||||
|
|
||||||
const specialCharText = 'Hello!@# world$%^ &*( this) is+ a-test';
|
const specialCharText = 'Hello!@# world$%^ &*( this) is+ a-test'
|
||||||
assert.deepEqual(
|
assert.deepEqual(splitter.splitText(specialCharText), [
|
||||||
splitter.splitText(specialCharText),
|
'Hello!@#',
|
||||||
['Hello!@#', 'world$%^', '&*( this)', 'is+', 'a-test']
|
'world$%^',
|
||||||
);
|
'&*( this)',
|
||||||
});
|
'is+',
|
||||||
|
'a-test',
|
||||||
|
])
|
||||||
|
})
|
||||||
|
|
||||||
it('Should handle chunkSize equal to chunkOverlap', () => {
|
it('Should handle chunkSize equal to chunkOverlap', () => {
|
||||||
splitter.chunkSize = 50;
|
splitter.chunkSize = 50
|
||||||
splitter.chunkOverlap = 50;
|
splitter.chunkOverlap = 50
|
||||||
assert.throws(
|
assert.throws(
|
||||||
() => splitter.splitText('Invalid configuration'),
|
() => splitter.splitText('Invalid configuration'),
|
||||||
new Error('Cannot have chunkOverlap >= chunkSize')
|
new Error('Cannot have chunkOverlap >= chunkSize'),
|
||||||
);
|
)
|
||||||
});
|
})
|
||||||
});
|
})
|
||||||
|
@ -1,60 +1,60 @@
|
|||||||
interface TextSplitterParams {
|
interface TextSplitterParams {
|
||||||
chunkSize: number;
|
chunkSize: number
|
||||||
|
|
||||||
chunkOverlap: number;
|
chunkOverlap: number
|
||||||
}
|
}
|
||||||
|
|
||||||
abstract class TextSplitter implements TextSplitterParams {
|
abstract class TextSplitter implements TextSplitterParams {
|
||||||
chunkSize = 1000;
|
chunkSize = 1000
|
||||||
chunkOverlap = 200;
|
chunkOverlap = 200
|
||||||
|
|
||||||
constructor(fields?: Partial<TextSplitterParams>) {
|
constructor(fields?: Partial<TextSplitterParams>) {
|
||||||
this.chunkSize = fields?.chunkSize ?? this.chunkSize;
|
this.chunkSize = fields?.chunkSize ?? this.chunkSize
|
||||||
this.chunkOverlap = fields?.chunkOverlap ?? this.chunkOverlap;
|
this.chunkOverlap = fields?.chunkOverlap ?? this.chunkOverlap
|
||||||
if (this.chunkOverlap >= this.chunkSize) {
|
if (this.chunkOverlap >= this.chunkSize) {
|
||||||
throw new Error('Cannot have chunkOverlap >= chunkSize');
|
throw new Error('Cannot have chunkOverlap >= chunkSize')
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
abstract splitText(text: string): string[];
|
abstract splitText(text: string): string[]
|
||||||
|
|
||||||
createDocuments(texts: string[]): string[] {
|
createDocuments(texts: string[]): string[] {
|
||||||
const documents: string[] = [];
|
const documents: string[] = []
|
||||||
for (let i = 0; i < texts.length; i += 1) {
|
for (let i = 0; i < texts.length; i += 1) {
|
||||||
const text = texts[i];
|
const text = texts[i]
|
||||||
for (const chunk of this.splitText(text!)) {
|
for (const chunk of this.splitText(text!)) {
|
||||||
documents.push(chunk);
|
documents.push(chunk)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return documents;
|
return documents
|
||||||
}
|
}
|
||||||
|
|
||||||
splitDocuments(documents: string[]): string[] {
|
splitDocuments(documents: string[]): string[] {
|
||||||
return this.createDocuments(documents);
|
return this.createDocuments(documents)
|
||||||
}
|
}
|
||||||
|
|
||||||
private joinDocs(docs: string[], separator: string): string | null {
|
private joinDocs(docs: string[], separator: string): string | null {
|
||||||
const text = docs.join(separator).trim();
|
const text = docs.join(separator).trim()
|
||||||
return text === '' ? null : text;
|
return text === '' ? null : text
|
||||||
}
|
}
|
||||||
|
|
||||||
mergeSplits(splits: string[], separator: string): string[] {
|
mergeSplits(splits: string[], separator: string): string[] {
|
||||||
const docs: string[] = [];
|
const docs: string[] = []
|
||||||
const currentDoc: string[] = [];
|
const currentDoc: string[] = []
|
||||||
let total = 0;
|
let total = 0
|
||||||
for (const d of splits) {
|
for (const d of splits) {
|
||||||
const _len = d.length;
|
const _len = d.length
|
||||||
if (total + _len >= this.chunkSize) {
|
if (total + _len >= this.chunkSize) {
|
||||||
if (total > this.chunkSize) {
|
if (total > this.chunkSize) {
|
||||||
console.warn(
|
console.warn(
|
||||||
`Created a chunk of size ${total}, +
|
`Created a chunk of size ${total}, +
|
||||||
which is longer than the specified ${this.chunkSize}`,
|
which is longer than the specified ${this.chunkSize}`,
|
||||||
);
|
)
|
||||||
}
|
}
|
||||||
if (currentDoc.length > 0) {
|
if (currentDoc.length > 0) {
|
||||||
const doc = this.joinDocs(currentDoc, separator);
|
const doc = this.joinDocs(currentDoc, separator)
|
||||||
if (doc !== null) {
|
if (doc !== null) {
|
||||||
docs.push(doc);
|
docs.push(doc)
|
||||||
}
|
}
|
||||||
// Keep on popping if:
|
// Keep on popping if:
|
||||||
// - we have a larger chunk than in the chunk overlap
|
// - we have a larger chunk than in the chunk overlap
|
||||||
@ -63,81 +63,81 @@ which is longer than the specified ${this.chunkSize}`,
|
|||||||
total > this.chunkOverlap ||
|
total > this.chunkOverlap ||
|
||||||
(total + _len > this.chunkSize && total > 0)
|
(total + _len > this.chunkSize && total > 0)
|
||||||
) {
|
) {
|
||||||
total -= currentDoc[0]!.length;
|
total -= currentDoc[0]!.length
|
||||||
currentDoc.shift();
|
currentDoc.shift()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
currentDoc.push(d);
|
currentDoc.push(d)
|
||||||
total += _len;
|
total += _len
|
||||||
}
|
}
|
||||||
const doc = this.joinDocs(currentDoc, separator);
|
const doc = this.joinDocs(currentDoc, separator)
|
||||||
if (doc !== null) {
|
if (doc !== null) {
|
||||||
docs.push(doc);
|
docs.push(doc)
|
||||||
}
|
}
|
||||||
return docs;
|
return docs
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RecursiveCharacterTextSplitterParams
|
export interface RecursiveCharacterTextSplitterParams
|
||||||
extends TextSplitterParams {
|
extends TextSplitterParams {
|
||||||
separators: string[];
|
separators: string[]
|
||||||
}
|
}
|
||||||
|
|
||||||
export class RecursiveCharacterTextSplitter
|
export class RecursiveCharacterTextSplitter
|
||||||
extends TextSplitter
|
extends TextSplitter
|
||||||
implements RecursiveCharacterTextSplitterParams
|
implements RecursiveCharacterTextSplitterParams
|
||||||
{
|
{
|
||||||
separators: string[] = ['\n\n', '\n', '.', ',', '>', '<', ' ', ''];
|
separators: string[] = ['\n\n', '\n', '.', ',', '>', '<', ' ', '']
|
||||||
|
|
||||||
constructor(fields?: Partial<RecursiveCharacterTextSplitterParams>) {
|
constructor(fields?: Partial<RecursiveCharacterTextSplitterParams>) {
|
||||||
super(fields);
|
super(fields)
|
||||||
this.separators = fields?.separators ?? this.separators;
|
this.separators = fields?.separators ?? this.separators
|
||||||
}
|
}
|
||||||
|
|
||||||
splitText(text: string): string[] {
|
splitText(text: string): string[] {
|
||||||
const finalChunks: string[] = [];
|
const finalChunks: string[] = []
|
||||||
|
|
||||||
// Get appropriate separator to use
|
// Get appropriate separator to use
|
||||||
let separator: string = this.separators[this.separators.length - 1]!;
|
let separator: string = this.separators[this.separators.length - 1]!
|
||||||
for (const s of this.separators) {
|
for (const s of this.separators) {
|
||||||
if (s === '') {
|
if (s === '') {
|
||||||
separator = s;
|
separator = s
|
||||||
break;
|
break
|
||||||
}
|
}
|
||||||
if (text.includes(s)) {
|
if (text.includes(s)) {
|
||||||
separator = s;
|
separator = s
|
||||||
break;
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Now that we have the separator, split the text
|
// Now that we have the separator, split the text
|
||||||
let splits: string[];
|
let splits: string[]
|
||||||
if (separator) {
|
if (separator) {
|
||||||
splits = text.split(separator);
|
splits = text.split(separator)
|
||||||
} else {
|
} else {
|
||||||
splits = text.split('');
|
splits = text.split('')
|
||||||
}
|
}
|
||||||
|
|
||||||
// Now go merging things, recursively splitting longer texts.
|
// Now go merging things, recursively splitting longer texts.
|
||||||
let goodSplits: string[] = [];
|
let goodSplits: string[] = []
|
||||||
for (const s of splits) {
|
for (const s of splits) {
|
||||||
if (s.length < this.chunkSize) {
|
if (s.length < this.chunkSize) {
|
||||||
goodSplits.push(s);
|
goodSplits.push(s)
|
||||||
} else {
|
} else {
|
||||||
if (goodSplits.length) {
|
if (goodSplits.length) {
|
||||||
const mergedText = this.mergeSplits(goodSplits, separator);
|
const mergedText = this.mergeSplits(goodSplits, separator)
|
||||||
finalChunks.push(...mergedText);
|
finalChunks.push(...mergedText)
|
||||||
goodSplits = [];
|
goodSplits = []
|
||||||
}
|
}
|
||||||
const otherInfo = this.splitText(s);
|
const otherInfo = this.splitText(s)
|
||||||
finalChunks.push(...otherInfo);
|
finalChunks.push(...otherInfo)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (goodSplits.length) {
|
if (goodSplits.length) {
|
||||||
const mergedText = this.mergeSplits(goodSplits, separator);
|
const mergedText = this.mergeSplits(goodSplits, separator)
|
||||||
finalChunks.push(...mergedText);
|
finalChunks.push(...mergedText)
|
||||||
}
|
}
|
||||||
return finalChunks;
|
return finalChunks
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1,43 +1,57 @@
|
|||||||
import { generateObject, streamText } from 'ai';
|
import { generateObject, streamText } from 'ai'
|
||||||
import { compact } from 'lodash-es';
|
import { compact } from 'lodash-es'
|
||||||
import pLimit from 'p-limit';
|
import pLimit from 'p-limit'
|
||||||
import { z } from 'zod';
|
import { z } from 'zod'
|
||||||
import { parseStreamingJson, type DeepPartial } from '~/utils/json';
|
import { parseStreamingJson, type DeepPartial } from '~/utils/json'
|
||||||
|
|
||||||
import { o3MiniModel, trimPrompt } from './ai/providers';
|
import { o3MiniModel, trimPrompt } from './ai/providers'
|
||||||
import { systemPrompt } from './prompt';
|
import { systemPrompt } from './prompt'
|
||||||
import zodToJsonSchema from 'zod-to-json-schema';
|
import zodToJsonSchema from 'zod-to-json-schema'
|
||||||
import { tavily, type TavilySearchResponse } from '@tavily/core';
|
import { tavily, type TavilySearchResponse } from '@tavily/core'
|
||||||
|
|
||||||
export type ResearchResult = {
|
export type ResearchResult = {
|
||||||
learnings: string[];
|
learnings: string[]
|
||||||
visitedUrls: string[];
|
visitedUrls: string[]
|
||||||
};
|
}
|
||||||
|
|
||||||
|
|
||||||
export interface WriteFinalReportParams {
|
export interface WriteFinalReportParams {
|
||||||
prompt: string;
|
prompt: string
|
||||||
learnings: string[];
|
learnings: string[]
|
||||||
}
|
}
|
||||||
// useRuntimeConfig()
|
// useRuntimeConfig()
|
||||||
// Used for streaming response
|
// Used for streaming response
|
||||||
export type SearchQuery = z.infer<typeof searchQueriesTypeSchema>['queries'][0];
|
export type SearchQuery = z.infer<typeof searchQueriesTypeSchema>['queries'][0]
|
||||||
export type PartialSearchQuery = DeepPartial<SearchQuery>;
|
export type PartialSearchQuery = DeepPartial<SearchQuery>
|
||||||
export type SearchResult = z.infer<typeof searchResultTypeSchema>;
|
export type SearchResult = z.infer<typeof searchResultTypeSchema>
|
||||||
export type PartialSearchResult = DeepPartial<SearchResult>;
|
export type PartialSearchResult = DeepPartial<SearchResult>
|
||||||
|
|
||||||
export type ResearchStep =
|
export type ResearchStep =
|
||||||
| { type: 'generating_query'; result: PartialSearchQuery; nodeId: string }
|
| { type: 'generating_query'; result: PartialSearchQuery; nodeId: string }
|
||||||
| { type: 'generated_query'; query: string; result: PartialSearchQuery; nodeId: string }
|
| {
|
||||||
|
type: 'generated_query'
|
||||||
|
query: string
|
||||||
|
result: PartialSearchQuery
|
||||||
|
nodeId: string
|
||||||
|
}
|
||||||
| { type: 'searching'; query: string; nodeId: string }
|
| { type: 'searching'; query: string; nodeId: string }
|
||||||
| { type: 'search_complete'; urls: string[]; nodeId: string }
|
| { type: 'search_complete'; urls: string[]; nodeId: string }
|
||||||
| { type: 'processing_serach_result'; query: string; result: PartialSearchResult; nodeId: string }
|
| {
|
||||||
| { type: 'processed_search_result'; query: string; result: SearchResult; nodeId: string }
|
type: 'processing_serach_result'
|
||||||
|
query: string
|
||||||
|
result: PartialSearchResult
|
||||||
|
nodeId: string
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
type: 'processed_search_result'
|
||||||
|
query: string
|
||||||
|
result: SearchResult
|
||||||
|
nodeId: string
|
||||||
|
}
|
||||||
| { type: 'error'; message: string; nodeId: string }
|
| { type: 'error'; message: string; nodeId: string }
|
||||||
| { type: 'complete'; learnings: string[], visitedUrls: string[] };
|
| { type: 'complete'; learnings: string[]; visitedUrls: string[] }
|
||||||
|
|
||||||
// increase this if you have higher API rate limits
|
// increase this if you have higher API rate limits
|
||||||
const ConcurrencyLimit = 2;
|
const ConcurrencyLimit = 2
|
||||||
|
|
||||||
// Initialize Firecrawl with optional API key and optional base url
|
// Initialize Firecrawl with optional API key and optional base url
|
||||||
|
|
||||||
@ -59,7 +73,7 @@ export const searchQueriesTypeSchema = z.object({
|
|||||||
researchGoal: z.string(),
|
researchGoal: z.string(),
|
||||||
}),
|
}),
|
||||||
),
|
),
|
||||||
});
|
})
|
||||||
|
|
||||||
// take en user query, return a list of SERP queries
|
// take en user query, return a list of SERP queries
|
||||||
export function generateSearchQueries({
|
export function generateSearchQueries({
|
||||||
@ -67,10 +81,10 @@ export function generateSearchQueries({
|
|||||||
numQueries = 3,
|
numQueries = 3,
|
||||||
learnings,
|
learnings,
|
||||||
}: {
|
}: {
|
||||||
query: string;
|
query: string
|
||||||
numQueries?: number;
|
numQueries?: number
|
||||||
// optional, if provided, the research will continue from the last learning
|
// optional, if provided, the research will continue from the last learning
|
||||||
learnings?: string[];
|
learnings?: string[]
|
||||||
}) {
|
}) {
|
||||||
const schema = z.object({
|
const schema = z.object({
|
||||||
queries: z
|
queries: z
|
||||||
@ -84,40 +98,38 @@ export function generateSearchQueries({
|
|||||||
),
|
),
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
.describe(`List of SERP queries, max of ${numQueries}`)
|
.describe(`List of SERP queries, max of ${numQueries}`),
|
||||||
})
|
})
|
||||||
const jsonSchema = JSON.stringify(zodToJsonSchema(schema));
|
const jsonSchema = JSON.stringify(zodToJsonSchema(schema))
|
||||||
|
|
||||||
const prompt = [
|
const prompt = [
|
||||||
`Given the following prompt from the user, generate a list of SERP queries to research the topic. Return a maximum of ${numQueries} queries, but feel free to return less if the original prompt is clear. Make sure each query is unique and not similar to each other: <prompt>${query}</prompt>\n\n`,
|
`Given the following prompt from the user, generate a list of SERP queries to research the topic. Return a maximum of ${numQueries} queries, but feel free to return less if the original prompt is clear. Make sure each query is unique and not similar to each other: <prompt>${query}</prompt>\n\n`,
|
||||||
learnings
|
learnings
|
||||||
? `Here are some learnings from previous research, use them to generate more specific queries: ${learnings.join(
|
? `Here are some learnings from previous research, use them to generate more specific queries: ${learnings.join('\n')}`
|
||||||
'\n',
|
|
||||||
)}`
|
|
||||||
: '',
|
: '',
|
||||||
`You MUST respond in JSON with the following schema: ${jsonSchema}`,
|
`You MUST respond in JSON with the following schema: ${jsonSchema}`,
|
||||||
].join('\n\n');
|
].join('\n\n')
|
||||||
return streamText({
|
return streamText({
|
||||||
model: o3MiniModel,
|
model: o3MiniModel,
|
||||||
system: systemPrompt(),
|
system: systemPrompt(),
|
||||||
prompt,
|
prompt,
|
||||||
});
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
export const searchResultTypeSchema = z.object({
|
export const searchResultTypeSchema = z.object({
|
||||||
learnings: z.array(z.string()),
|
learnings: z.array(z.string()),
|
||||||
followUpQuestions: z.array(z.string()),
|
followUpQuestions: z.array(z.string()),
|
||||||
});
|
})
|
||||||
function processSearchResult({
|
function processSearchResult({
|
||||||
query,
|
query,
|
||||||
result,
|
result,
|
||||||
numLearnings = 3,
|
numLearnings = 3,
|
||||||
numFollowUpQuestions = 3,
|
numFollowUpQuestions = 3,
|
||||||
}: {
|
}: {
|
||||||
query: string;
|
query: string
|
||||||
result: TavilySearchResponse
|
result: TavilySearchResponse
|
||||||
numLearnings?: number;
|
numLearnings?: number
|
||||||
numFollowUpQuestions?: number;
|
numFollowUpQuestions?: number
|
||||||
}) {
|
}) {
|
||||||
const schema = z.object({
|
const schema = z.object({
|
||||||
learnings: z
|
learnings: z
|
||||||
@ -128,25 +140,23 @@ function processSearchResult({
|
|||||||
.describe(
|
.describe(
|
||||||
`List of follow-up questions to research the topic further, max of ${numFollowUpQuestions}`,
|
`List of follow-up questions to research the topic further, max of ${numFollowUpQuestions}`,
|
||||||
),
|
),
|
||||||
});
|
})
|
||||||
const jsonSchema = JSON.stringify(zodToJsonSchema(schema));
|
const jsonSchema = JSON.stringify(zodToJsonSchema(schema))
|
||||||
const contents = compact(result.results.map(item => item.content)).map(
|
const contents = compact(result.results.map((item) => item.content)).map(
|
||||||
content => trimPrompt(content, 25_000),
|
(content) => trimPrompt(content, 25_000),
|
||||||
);
|
)
|
||||||
const prompt = [
|
const prompt = [
|
||||||
`Given the following contents from a SERP search for the query <query>${query}</query>, generate a list of learnings from the contents. Return a maximum of ${numLearnings} learnings, but feel free to return less if the contents are clear. Make sure each learning is unique and not similar to each other. The learnings should be concise and to the point, as detailed and information dense as possible. Make sure to include any entities like people, places, companies, products, things, etc in the learnings, as well as any exact metrics, numbers, or dates. The learnings will be used to research the topic further.`,
|
`Given the following contents from a SERP search for the query <query>${query}</query>, generate a list of learnings from the contents. Return a maximum of ${numLearnings} learnings, but feel free to return less if the contents are clear. Make sure each learning is unique and not similar to each other. The learnings should be concise and to the point, as detailed and information dense as possible. Make sure to include any entities like people, places, companies, products, things, etc in the learnings, as well as any exact metrics, numbers, or dates. The learnings will be used to research the topic further.`,
|
||||||
`<contents>${contents
|
`<contents>${contents.map((content) => `<content>\n${content}\n</content>`).join('\n')}</contents>`,
|
||||||
.map(content => `<content>\n${content}\n</content>`)
|
|
||||||
.join('\n')}</contents>`,
|
|
||||||
`You MUST respond in JSON with the following schema: ${jsonSchema}`,
|
`You MUST respond in JSON with the following schema: ${jsonSchema}`,
|
||||||
].join('\n\n');
|
].join('\n\n')
|
||||||
|
|
||||||
return streamText({
|
return streamText({
|
||||||
model: o3MiniModel,
|
model: o3MiniModel,
|
||||||
abortSignal: AbortSignal.timeout(60_000),
|
abortSignal: AbortSignal.timeout(60_000),
|
||||||
system: systemPrompt(),
|
system: systemPrompt(),
|
||||||
prompt,
|
prompt,
|
||||||
});
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
export function writeFinalReport({
|
export function writeFinalReport({
|
||||||
@ -155,28 +165,28 @@ export function writeFinalReport({
|
|||||||
}: WriteFinalReportParams) {
|
}: WriteFinalReportParams) {
|
||||||
const learningsString = trimPrompt(
|
const learningsString = trimPrompt(
|
||||||
learnings
|
learnings
|
||||||
.map(learning => `<learning>\n${learning}\n</learning>`)
|
.map((learning) => `<learning>\n${learning}\n</learning>`)
|
||||||
.join('\n'),
|
.join('\n'),
|
||||||
150_000,
|
150_000,
|
||||||
);
|
)
|
||||||
const _prompt = [
|
const _prompt = [
|
||||||
`Given the following prompt from the user, write a final report on the topic using the learnings from research. Make it as as detailed as possible, aim for 3 or more pages, include ALL the learnings from research:`,
|
`Given the following prompt from the user, write a final report on the topic using the learnings from research. Make it as as detailed as possible, aim for 3 or more pages, include ALL the learnings from research:`,
|
||||||
`<prompt>${prompt}</prompt>`,
|
`<prompt>${prompt}</prompt>`,
|
||||||
`Here are all the learnings from previous research:`,
|
`Here are all the learnings from previous research:`,
|
||||||
`<learnings>\n${learningsString}\n</learnings>`,
|
`<learnings>\n${learningsString}\n</learnings>`,
|
||||||
`Write the report in Markdown.`,
|
`Write the report in Markdown.`,
|
||||||
`## Deep Research Report`
|
`## Deep Research Report`,
|
||||||
].join('\n\n');
|
].join('\n\n')
|
||||||
|
|
||||||
return streamText({
|
return streamText({
|
||||||
model: o3MiniModel,
|
model: o3MiniModel,
|
||||||
system: systemPrompt(),
|
system: systemPrompt(),
|
||||||
prompt: _prompt,
|
prompt: _prompt,
|
||||||
});
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
function childNodeId(parentNodeId: string, currentIndex: number) {
|
function childNodeId(parentNodeId: string, currentIndex: number) {
|
||||||
return `${parentNodeId}-${currentIndex}`;
|
return `${parentNodeId}-${currentIndex}`
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function deepResearch({
|
export async function deepResearch({
|
||||||
@ -187,15 +197,15 @@ export async function deepResearch({
|
|||||||
visitedUrls = [],
|
visitedUrls = [],
|
||||||
onProgress,
|
onProgress,
|
||||||
currentDepth = 1,
|
currentDepth = 1,
|
||||||
nodeId = '0'
|
nodeId = '0',
|
||||||
}: {
|
}: {
|
||||||
query: string;
|
query: string
|
||||||
breadth: number;
|
breadth: number
|
||||||
maxDepth: number;
|
maxDepth: number
|
||||||
learnings?: string[];
|
learnings?: string[]
|
||||||
visitedUrls?: string[];
|
visitedUrls?: string[]
|
||||||
onProgress: (step: ResearchStep) => void;
|
onProgress: (step: ResearchStep) => void
|
||||||
currentDepth?: number;
|
currentDepth?: number
|
||||||
nodeId?: string
|
nodeId?: string
|
||||||
}): Promise<ResearchResult> {
|
}): Promise<ResearchResult> {
|
||||||
try {
|
try {
|
||||||
@ -203,25 +213,25 @@ export async function deepResearch({
|
|||||||
query,
|
query,
|
||||||
learnings,
|
learnings,
|
||||||
numQueries: breadth,
|
numQueries: breadth,
|
||||||
});
|
})
|
||||||
const limit = pLimit(ConcurrencyLimit);
|
const limit = pLimit(ConcurrencyLimit)
|
||||||
|
|
||||||
let searchQueries: PartialSearchQuery[] = [];
|
let searchQueries: PartialSearchQuery[] = []
|
||||||
|
|
||||||
for await (const parsedQueries of parseStreamingJson(
|
for await (const parsedQueries of parseStreamingJson(
|
||||||
searchQueriesResult.textStream,
|
searchQueriesResult.textStream,
|
||||||
searchQueriesTypeSchema,
|
searchQueriesTypeSchema,
|
||||||
(value) => !!value.queries?.length && !!value.queries[0]?.query
|
(value) => !!value.queries?.length && !!value.queries[0]?.query,
|
||||||
)) {
|
)) {
|
||||||
if (parsedQueries.queries) {
|
if (parsedQueries.queries) {
|
||||||
for (let i = 0; i < searchQueries.length; i++) {
|
for (let i = 0; i < searchQueries.length; i++) {
|
||||||
onProgress({
|
onProgress({
|
||||||
type: 'generating_query',
|
type: 'generating_query',
|
||||||
result: searchQueries[i],
|
result: searchQueries[i],
|
||||||
nodeId: childNodeId(nodeId, i)
|
nodeId: childNodeId(nodeId, i),
|
||||||
});
|
})
|
||||||
}
|
}
|
||||||
searchQueries = parsedQueries.queries;
|
searchQueries = parsedQueries.queries
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -230,21 +240,22 @@ export async function deepResearch({
|
|||||||
type: 'generated_query',
|
type: 'generated_query',
|
||||||
query,
|
query,
|
||||||
result: searchQueries[i],
|
result: searchQueries[i],
|
||||||
nodeId: childNodeId(nodeId, i)
|
nodeId: childNodeId(nodeId, i),
|
||||||
});
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
const results = await Promise.all(
|
const results = await Promise.all(
|
||||||
searchQueries.map((searchQuery, i) =>
|
searchQueries.map((searchQuery, i) =>
|
||||||
limit(async () => {
|
limit(async () => {
|
||||||
if (!searchQuery?.query) return {
|
if (!searchQuery?.query)
|
||||||
learnings: [],
|
return {
|
||||||
visitedUrls: [],
|
learnings: [],
|
||||||
}
|
visitedUrls: [],
|
||||||
|
}
|
||||||
onProgress({
|
onProgress({
|
||||||
type: 'searching',
|
type: 'searching',
|
||||||
query: searchQuery.query,
|
query: searchQuery.query,
|
||||||
nodeId: childNodeId(nodeId, i)
|
nodeId: childNodeId(nodeId, i),
|
||||||
})
|
})
|
||||||
try {
|
try {
|
||||||
// const result = await firecrawl.search(searchQuery.query, {
|
// const result = await firecrawl.search(searchQuery.query, {
|
||||||
@ -255,42 +266,50 @@ export async function deepResearch({
|
|||||||
const result = await tvly.search(searchQuery.query, {
|
const result = await tvly.search(searchQuery.query, {
|
||||||
maxResults: 5,
|
maxResults: 5,
|
||||||
})
|
})
|
||||||
console.log(`Ran ${searchQuery.query}, found ${result.results.length} contents`);
|
console.log(
|
||||||
|
`Ran ${searchQuery.query}, found ${result.results.length} contents`,
|
||||||
|
)
|
||||||
|
|
||||||
// Collect URLs from this search
|
// Collect URLs from this search
|
||||||
const newUrls = compact(result.results.map(item => item.url));
|
const newUrls = compact(result.results.map((item) => item.url))
|
||||||
onProgress({
|
onProgress({
|
||||||
type: 'search_complete',
|
type: 'search_complete',
|
||||||
urls: newUrls,
|
urls: newUrls,
|
||||||
nodeId: childNodeId(nodeId, i),
|
nodeId: childNodeId(nodeId, i),
|
||||||
})
|
})
|
||||||
// Breadth for the next search is half of the current breadth
|
// Breadth for the next search is half of the current breadth
|
||||||
const nextBreadth = Math.ceil(breadth / 2);
|
const nextBreadth = Math.ceil(breadth / 2)
|
||||||
|
|
||||||
const searchResultGenerator = processSearchResult({
|
const searchResultGenerator = processSearchResult({
|
||||||
query: searchQuery.query,
|
query: searchQuery.query,
|
||||||
result,
|
result,
|
||||||
numFollowUpQuestions: nextBreadth,
|
numFollowUpQuestions: nextBreadth,
|
||||||
});
|
})
|
||||||
let searchResult: PartialSearchResult = {};
|
let searchResult: PartialSearchResult = {}
|
||||||
|
|
||||||
for await (const parsedLearnings of parseStreamingJson(
|
for await (const parsedLearnings of parseStreamingJson(
|
||||||
searchResultGenerator.textStream,
|
searchResultGenerator.textStream,
|
||||||
searchResultTypeSchema,
|
searchResultTypeSchema,
|
||||||
(value) => !!value.learnings?.length
|
(value) => !!value.learnings?.length,
|
||||||
)) {
|
)) {
|
||||||
searchResult = parsedLearnings;
|
searchResult = parsedLearnings
|
||||||
onProgress({
|
onProgress({
|
||||||
type: 'processing_serach_result',
|
type: 'processing_serach_result',
|
||||||
result: parsedLearnings,
|
result: parsedLearnings,
|
||||||
query: searchQuery.query,
|
query: searchQuery.query,
|
||||||
nodeId: childNodeId(nodeId, i)
|
nodeId: childNodeId(nodeId, i),
|
||||||
});
|
})
|
||||||
}
|
}
|
||||||
console.log(`Processed search result for ${searchQuery.query}`, searchResult);
|
console.log(
|
||||||
const allLearnings = [...learnings, ...(searchResult.learnings ?? [])];
|
`Processed search result for ${searchQuery.query}`,
|
||||||
const allUrls = [...visitedUrls, ...newUrls];
|
searchResult,
|
||||||
const nextDepth = currentDepth + 1;
|
)
|
||||||
|
const allLearnings = [
|
||||||
|
...learnings,
|
||||||
|
...(searchResult.learnings ?? []),
|
||||||
|
]
|
||||||
|
const allUrls = [...visitedUrls, ...newUrls]
|
||||||
|
const nextDepth = currentDepth + 1
|
||||||
|
|
||||||
onProgress({
|
onProgress({
|
||||||
type: 'processed_search_result',
|
type: 'processed_search_result',
|
||||||
@ -299,18 +318,21 @@ export async function deepResearch({
|
|||||||
followUpQuestions: searchResult.followUpQuestions ?? [],
|
followUpQuestions: searchResult.followUpQuestions ?? [],
|
||||||
},
|
},
|
||||||
query: searchQuery.query,
|
query: searchQuery.query,
|
||||||
nodeId: childNodeId(nodeId, i)
|
nodeId: childNodeId(nodeId, i),
|
||||||
})
|
})
|
||||||
|
|
||||||
if (nextDepth < maxDepth && searchResult.followUpQuestions?.length) {
|
if (
|
||||||
|
nextDepth < maxDepth &&
|
||||||
|
searchResult.followUpQuestions?.length
|
||||||
|
) {
|
||||||
console.warn(
|
console.warn(
|
||||||
`Researching deeper, breadth: ${nextBreadth}, depth: ${nextDepth}`,
|
`Researching deeper, breadth: ${nextBreadth}, depth: ${nextDepth}`,
|
||||||
);
|
)
|
||||||
|
|
||||||
const nextQuery = `
|
const nextQuery = `
|
||||||
Previous research goal: ${searchQuery.researchGoal}
|
Previous research goal: ${searchQuery.researchGoal}
|
||||||
Follow-up research directions: ${searchResult.followUpQuestions.map(q => `\n${q}`).join('')}
|
Follow-up research directions: ${searchResult.followUpQuestions.map((q) => `\n${q}`).join('')}
|
||||||
`.trim();
|
`.trim()
|
||||||
|
|
||||||
return deepResearch({
|
return deepResearch({
|
||||||
query: nextQuery,
|
query: nextQuery,
|
||||||
@ -321,36 +343,38 @@ export async function deepResearch({
|
|||||||
onProgress,
|
onProgress,
|
||||||
currentDepth: nextDepth,
|
currentDepth: nextDepth,
|
||||||
nodeId: childNodeId(nodeId, i),
|
nodeId: childNodeId(nodeId, i),
|
||||||
});
|
})
|
||||||
} else {
|
} else {
|
||||||
return {
|
return {
|
||||||
learnings: allLearnings,
|
learnings: allLearnings,
|
||||||
visitedUrls: allUrls,
|
visitedUrls: allUrls,
|
||||||
};
|
}
|
||||||
}
|
}
|
||||||
} catch (e: any) {
|
} catch (e: any) {
|
||||||
throw new Error(`Error searching for ${searchQuery.query}, depth ${currentDepth}\nMessage: ${e.message}`)
|
throw new Error(
|
||||||
|
`Error searching for ${searchQuery.query}, depth ${currentDepth}\nMessage: ${e.message}`,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}),
|
}),
|
||||||
),
|
),
|
||||||
);
|
)
|
||||||
// Conclude results
|
// Conclude results
|
||||||
const _learnings = [...new Set(results.flatMap(r => r.learnings))]
|
const _learnings = [...new Set(results.flatMap((r) => r.learnings))]
|
||||||
const _visitedUrls = [...new Set(results.flatMap(r => r.visitedUrls))]
|
const _visitedUrls = [...new Set(results.flatMap((r) => r.visitedUrls))]
|
||||||
// Complete should only be called once
|
// Complete should only be called once
|
||||||
if (nodeId === '0') {
|
if (nodeId === '0') {
|
||||||
onProgress({
|
onProgress({
|
||||||
type: 'complete',
|
type: 'complete',
|
||||||
learnings: _learnings,
|
learnings: _learnings,
|
||||||
visitedUrls: _visitedUrls,
|
visitedUrls: _visitedUrls,
|
||||||
});
|
})
|
||||||
}
|
}
|
||||||
return {
|
return {
|
||||||
learnings: _learnings,
|
learnings: _learnings,
|
||||||
visitedUrls: _visitedUrls,
|
visitedUrls: _visitedUrls,
|
||||||
}
|
}
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
console.error(error);
|
console.error(error)
|
||||||
onProgress({
|
onProgress({
|
||||||
type: 'error',
|
type: 'error',
|
||||||
message: error?.message ?? 'Something went wrong',
|
message: error?.message ?? 'Something went wrong',
|
||||||
@ -361,4 +385,4 @@ export async function deepResearch({
|
|||||||
visitedUrls: [],
|
visitedUrls: [],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1,22 +1,22 @@
|
|||||||
import { streamText } from 'ai';
|
import { streamText } from 'ai'
|
||||||
import { z } from 'zod';
|
import { z } from 'zod'
|
||||||
import { zodToJsonSchema } from 'zod-to-json-schema'
|
import { zodToJsonSchema } from 'zod-to-json-schema'
|
||||||
|
|
||||||
import { o3MiniModel } from './ai/providers';
|
import { o3MiniModel } from './ai/providers'
|
||||||
import { systemPrompt } from './prompt';
|
import { systemPrompt } from './prompt'
|
||||||
|
|
||||||
type PartialFeedback = DeepPartial<z.infer<typeof feedbackTypeSchema>>
|
type PartialFeedback = DeepPartial<z.infer<typeof feedbackTypeSchema>>
|
||||||
|
|
||||||
export const feedbackTypeSchema = z.object({
|
export const feedbackTypeSchema = z.object({
|
||||||
questions: z.array(z.string())
|
questions: z.array(z.string()),
|
||||||
})
|
})
|
||||||
|
|
||||||
export function generateFeedback({
|
export function generateFeedback({
|
||||||
query,
|
query,
|
||||||
numQuestions = 3,
|
numQuestions = 3,
|
||||||
}: {
|
}: {
|
||||||
query: string;
|
query: string
|
||||||
numQuestions?: number;
|
numQuestions?: number
|
||||||
}) {
|
}) {
|
||||||
const schema = z.object({
|
const schema = z.object({
|
||||||
questions: z
|
questions: z
|
||||||
@ -24,22 +24,22 @@ export function generateFeedback({
|
|||||||
.describe(
|
.describe(
|
||||||
`Follow up questions to clarify the research direction, max of ${numQuestions}`,
|
`Follow up questions to clarify the research direction, max of ${numQuestions}`,
|
||||||
),
|
),
|
||||||
});
|
})
|
||||||
const jsonSchema = JSON.stringify(zodToJsonSchema(schema));
|
const jsonSchema = JSON.stringify(zodToJsonSchema(schema))
|
||||||
const prompt = [
|
const prompt = [
|
||||||
`Given the following query from the user, ask some follow up questions to clarify the research direction. Return a maximum of ${numQuestions} questions, but feel free to return less if the original query is clear: <query>${query}</query>`,
|
`Given the following query from the user, ask some follow up questions to clarify the research direction. Return a maximum of ${numQuestions} questions, but feel free to return less if the original query is clear: <query>${query}</query>`,
|
||||||
`You MUST respond in JSON with the following schema: ${jsonSchema}`,
|
`You MUST respond in JSON with the following schema: ${jsonSchema}`,
|
||||||
].join('\n\n');
|
].join('\n\n')
|
||||||
|
|
||||||
const stream = streamText({
|
const stream = streamText({
|
||||||
model: o3MiniModel,
|
model: o3MiniModel,
|
||||||
system: systemPrompt(),
|
system: systemPrompt(),
|
||||||
prompt,
|
prompt,
|
||||||
});
|
})
|
||||||
|
|
||||||
return parseStreamingJson(
|
return parseStreamingJson(
|
||||||
stream.textStream,
|
stream.textStream,
|
||||||
feedbackTypeSchema,
|
feedbackTypeSchema,
|
||||||
(value: PartialFeedback) => !!value.questions && value.questions.length > 0
|
(value: PartialFeedback) => !!value.questions && value.questions.length > 0,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
@ -1,5 +1,5 @@
|
|||||||
export const systemPrompt = () => {
|
export const systemPrompt = () => {
|
||||||
const now = new Date().toISOString();
|
const now = new Date().toISOString()
|
||||||
return `You are an expert researcher. Today is ${now}. Follow these instructions when responding:
|
return `You are an expert researcher. Today is ${now}. Follow these instructions when responding:
|
||||||
- You may be asked to research subjects that is after your knowledge cutoff, assume the user is right when presented with news.
|
- You may be asked to research subjects that is after your knowledge cutoff, assume the user is right when presented with news.
|
||||||
- The user is a highly experienced analyst, no need to simplify it, be as detailed as possible and make sure your response is correct.
|
- The user is a highly experienced analyst, no need to simplify it, be as detailed as possible and make sure your response is correct.
|
||||||
@ -11,5 +11,5 @@ export const systemPrompt = () => {
|
|||||||
- Provide detailed explanations, I'm comfortable with lots of detail.
|
- Provide detailed explanations, I'm comfortable with lots of detail.
|
||||||
- Value good arguments over authorities, the source is irrelevant.
|
- Value good arguments over authorities, the source is irrelevant.
|
||||||
- Consider new technologies and contrarian ideas, not just the conventional wisdom.
|
- Consider new technologies and contrarian ideas, not just the conventional wisdom.
|
||||||
- You may use high levels of speculation or prediction, just flag it for me.`;
|
- You may use high levels of speculation or prediction, just flag it for me.`
|
||||||
};
|
}
|
||||||
|
62
lib/run.ts
62
lib/run.ts
@ -1,27 +1,27 @@
|
|||||||
import * as fs from 'fs/promises';
|
import * as fs from 'fs/promises'
|
||||||
import * as readline from 'readline';
|
import * as readline from 'readline'
|
||||||
|
|
||||||
import { deepResearch, writeFinalReport } from './deep-research';
|
import { deepResearch, writeFinalReport } from './deep-research'
|
||||||
import { generateFeedback } from './feedback';
|
import { generateFeedback } from './feedback'
|
||||||
|
|
||||||
const rl = readline.createInterface({
|
const rl = readline.createInterface({
|
||||||
input: process.stdin,
|
input: process.stdin,
|
||||||
output: process.stdout,
|
output: process.stdout,
|
||||||
});
|
})
|
||||||
|
|
||||||
// Helper function to get user input
|
// Helper function to get user input
|
||||||
function askQuestion(query: string): Promise<string> {
|
function askQuestion(query: string): Promise<string> {
|
||||||
return new Promise(resolve => {
|
return new Promise((resolve) => {
|
||||||
rl.question(query, answer => {
|
rl.question(query, (answer) => {
|
||||||
resolve(answer);
|
resolve(answer)
|
||||||
});
|
})
|
||||||
});
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// run the agent
|
// run the agent
|
||||||
async function run() {
|
async function run() {
|
||||||
// Get initial query
|
// Get initial query
|
||||||
const initialQuery = await askQuestion('What would you like to research? ');
|
const initialQuery = await askQuestion('What would you like to research? ')
|
||||||
|
|
||||||
// Get breath and depth parameters
|
// Get breath and depth parameters
|
||||||
const breadth =
|
const breadth =
|
||||||
@ -30,29 +30,29 @@ async function run() {
|
|||||||
'Enter research breadth (recommended 2-10, default 4): ',
|
'Enter research breadth (recommended 2-10, default 4): ',
|
||||||
),
|
),
|
||||||
10,
|
10,
|
||||||
) || 4;
|
) || 4
|
||||||
const depth =
|
const depth =
|
||||||
parseInt(
|
parseInt(
|
||||||
await askQuestion('Enter research depth (recommended 1-5, default 2): '),
|
await askQuestion('Enter research depth (recommended 1-5, default 2): '),
|
||||||
10,
|
10,
|
||||||
) || 2;
|
) || 2
|
||||||
|
|
||||||
console.log(`Creating research plan...`);
|
console.log(`Creating research plan...`)
|
||||||
|
|
||||||
// Generate follow-up questions
|
// Generate follow-up questions
|
||||||
const followUpQuestions = await generateFeedback({
|
const followUpQuestions = await generateFeedback({
|
||||||
query: initialQuery,
|
query: initialQuery,
|
||||||
});
|
})
|
||||||
|
|
||||||
console.log(
|
console.log(
|
||||||
'\nTo better understand your research needs, please answer these follow-up questions:',
|
'\nTo better understand your research needs, please answer these follow-up questions:',
|
||||||
);
|
)
|
||||||
|
|
||||||
// Collect answers to follow-up questions
|
// Collect answers to follow-up questions
|
||||||
const answers: string[] = [];
|
const answers: string[] = []
|
||||||
for (const question of followUpQuestions) {
|
for (const question of followUpQuestions) {
|
||||||
const answer = await askQuestion(`\n${question}\nYour answer: `);
|
const answer = await askQuestion(`\n${question}\nYour answer: `)
|
||||||
answers.push(answer);
|
answers.push(answer)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Combine all information for deep research
|
// Combine all information for deep research
|
||||||
@ -60,34 +60,34 @@ async function run() {
|
|||||||
Initial Query: ${initialQuery}
|
Initial Query: ${initialQuery}
|
||||||
Follow-up Questions and Answers:
|
Follow-up Questions and Answers:
|
||||||
${followUpQuestions.map((q, i) => `Q: ${q}\nA: ${answers[i]}`).join('\n')}
|
${followUpQuestions.map((q, i) => `Q: ${q}\nA: ${answers[i]}`).join('\n')}
|
||||||
`;
|
`
|
||||||
|
|
||||||
console.log('\nResearching your topic...');
|
console.log('\nResearching your topic...')
|
||||||
|
|
||||||
const { learnings, visitedUrls } = await deepResearch({
|
const { learnings, visitedUrls } = await deepResearch({
|
||||||
query: combinedQuery,
|
query: combinedQuery,
|
||||||
breadth,
|
breadth,
|
||||||
depth,
|
depth,
|
||||||
});
|
})
|
||||||
|
|
||||||
console.log(`\n\nLearnings:\n\n${learnings.join('\n')}`);
|
console.log(`\n\nLearnings:\n\n${learnings.join('\n')}`)
|
||||||
console.log(
|
console.log(
|
||||||
`\n\nVisited URLs (${visitedUrls.length}):\n\n${visitedUrls.join('\n')}`,
|
`\n\nVisited URLs (${visitedUrls.length}):\n\n${visitedUrls.join('\n')}`,
|
||||||
);
|
)
|
||||||
console.log('Writing final report...');
|
console.log('Writing final report...')
|
||||||
|
|
||||||
const report = await writeFinalReport({
|
const report = await writeFinalReport({
|
||||||
prompt: combinedQuery,
|
prompt: combinedQuery,
|
||||||
learnings,
|
learnings,
|
||||||
visitedUrls,
|
visitedUrls,
|
||||||
});
|
})
|
||||||
|
|
||||||
// Save report to file
|
// Save report to file
|
||||||
await fs.writeFile('output.md', report, 'utf-8');
|
await fs.writeFile('output.md', report, 'utf-8')
|
||||||
|
|
||||||
console.log(`\n\nFinal Report:\n\n${report}`);
|
console.log(`\n\nFinal Report:\n\n${report}`)
|
||||||
console.log('\nReport has been saved to output.md');
|
console.log('\nReport has been saved to output.md')
|
||||||
rl.close();
|
rl.close()
|
||||||
}
|
}
|
||||||
|
|
||||||
run().catch(console.error);
|
run().catch(console.error)
|
||||||
|
@ -1,11 +1,6 @@
|
|||||||
// https://nuxt.com/docs/api/configuration/nuxt-config
|
// https://nuxt.com/docs/api/configuration/nuxt-config
|
||||||
export default defineNuxtConfig({
|
export default defineNuxtConfig({
|
||||||
modules: [
|
modules: ['@pinia/nuxt', '@nuxt/ui', '@nuxtjs/color-mode', '@vueuse/nuxt'],
|
||||||
'@pinia/nuxt',
|
|
||||||
'@nuxt/ui',
|
|
||||||
'@nuxtjs/color-mode',
|
|
||||||
'@vueuse/nuxt',
|
|
||||||
],
|
|
||||||
|
|
||||||
colorMode: {
|
colorMode: {
|
||||||
preference: 'system',
|
preference: 'system',
|
||||||
|
@ -3,12 +3,26 @@
|
|||||||
<UContainer>
|
<UContainer>
|
||||||
<div class="max-w-4xl mx-auto py-8 space-y-4">
|
<div class="max-w-4xl mx-auto py-8 space-y-4">
|
||||||
<div class="flex items-center justify-between">
|
<div class="flex items-center justify-between">
|
||||||
<h1 class="text-3xl font-bold text-center mb-2"> Deep Research Assistant </h1>
|
<h1 class="text-3xl font-bold text-center mb-2">
|
||||||
|
Deep Research Assistant
|
||||||
|
</h1>
|
||||||
<ColorModeButton />
|
<ColorModeButton />
|
||||||
</div>
|
</div>
|
||||||
<ResearchForm :is-loading-feedback="!!feedbackRef?.isLoading" ref="formRef" @submit="generateFeedback" />
|
<ResearchForm
|
||||||
<ResearchFeedback :is-loading-search="!!deepResearchRef?.isLoading" ref="feedbackRef" @submit="startDeepSearch" />
|
:is-loading-feedback="!!feedbackRef?.isLoading"
|
||||||
<DeepResearch ref="deepResearchRef" @complete="generateReport" class="mb-8" />
|
ref="formRef"
|
||||||
|
@submit="generateFeedback"
|
||||||
|
/>
|
||||||
|
<ResearchFeedback
|
||||||
|
:is-loading-search="!!deepResearchRef?.isLoading"
|
||||||
|
ref="feedbackRef"
|
||||||
|
@submit="startDeepSearch"
|
||||||
|
/>
|
||||||
|
<DeepResearch
|
||||||
|
ref="deepResearchRef"
|
||||||
|
@complete="generateReport"
|
||||||
|
class="mb-8"
|
||||||
|
/>
|
||||||
<ResearchReport ref="reportRef" />
|
<ResearchReport ref="reportRef" />
|
||||||
</div>
|
</div>
|
||||||
</UContainer>
|
</UContainer>
|
||||||
@ -49,9 +63,18 @@ ${feedback.value.map((qa) => `Q: ${qa.assistantQuestion}\nA: ${qa.userAnswer}`).
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function startDeepSearch(_feedback: ResearchFeedbackResult[]) {
|
async function startDeepSearch(_feedback: ResearchFeedbackResult[]) {
|
||||||
if (!formRef.value?.form.query || !formRef.value?.form.breadth || !formRef.value?.form.depth) return
|
if (
|
||||||
|
!formRef.value?.form.query ||
|
||||||
|
!formRef.value?.form.breadth ||
|
||||||
|
!formRef.value?.form.depth
|
||||||
|
)
|
||||||
|
return
|
||||||
feedback.value = _feedback
|
feedback.value = _feedback
|
||||||
deepResearchRef.value?.startResearch(getCombinedQuery(), formRef.value.form.breadth, formRef.value.form.depth)
|
deepResearchRef.value?.startResearch(
|
||||||
|
getCombinedQuery(),
|
||||||
|
formRef.value.form.breadth,
|
||||||
|
formRef.value.form.depth,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
async function generateReport(_researchResult: ResearchResult) {
|
async function generateReport(_researchResult: ResearchResult) {
|
||||||
|
@ -6,5 +6,4 @@ module.exports = {
|
|||||||
proseWrap: 'never',
|
proseWrap: 'never',
|
||||||
htmlWhitespaceSensitivity: 'strict',
|
htmlWhitespaceSensitivity: 'strict',
|
||||||
endOfLine: 'auto',
|
endOfLine: 'auto',
|
||||||
printWidth: 140,
|
|
||||||
}
|
}
|
||||||
|
@ -1,9 +1,8 @@
|
|||||||
// This file is currently unused
|
// This file is currently unused
|
||||||
import { deepResearch, ResearchStep } from "~/lib/deep-research";
|
import { deepResearch, ResearchStep } from '~/lib/deep-research'
|
||||||
|
|
||||||
export default defineEventHandler(async event => {
|
export default defineEventHandler(async (event) => {
|
||||||
const { initialQuery, feedback, depth, breadth } =
|
const { initialQuery, feedback, depth, breadth } = await readBody(event)
|
||||||
await readBody(event)
|
|
||||||
console.log({ initialQuery, feedback, depth, breadth })
|
console.log({ initialQuery, feedback, depth, breadth })
|
||||||
|
|
||||||
// 设置 SSE 响应头
|
// 设置 SSE 响应头
|
||||||
@ -31,4 +30,4 @@ ${feedback.map((qa: { question: string; answer: string }) => `Q: ${qa.question}\
|
|||||||
})
|
})
|
||||||
resolve()
|
resolve()
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
// This file is currently unused
|
// This file is currently unused
|
||||||
import { generateFeedback } from "~/lib/feedback";
|
import { generateFeedback } from '~/lib/feedback'
|
||||||
|
|
||||||
export default defineEventHandler(async event => {
|
export default defineEventHandler(async (event) => {
|
||||||
const { query, numQuestions } = await readBody(event)
|
const { query, numQuestions } = await readBody(event)
|
||||||
console.log({ query, numQuestions })
|
console.log({ query, numQuestions })
|
||||||
const feedback = generateFeedback({
|
const feedback = generateFeedback({
|
||||||
@ -16,4 +16,4 @@ export default defineEventHandler(async event => {
|
|||||||
return 'Error generating feedback'
|
return 'Error generating feedback'
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
5
stores/config.ts
Normal file
5
stores/config.ts
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
export interface Config {
|
||||||
|
ai: {
|
||||||
|
provider: 'openai-compatible'
|
||||||
|
}
|
||||||
|
}
|
16
test.ts
Normal file
16
test.ts
Normal file
@ -0,0 +1,16 @@
|
|||||||
|
import { generateFeedback } from './lib/feedback'
|
||||||
|
|
||||||
|
async function test() {
|
||||||
|
console.log('test')
|
||||||
|
const feedback = generateFeedback({
|
||||||
|
query: 'Why is the sky blue?',
|
||||||
|
})
|
||||||
|
console.log('feedback', feedback)
|
||||||
|
|
||||||
|
for await (const partial of feedback.textStream) {
|
||||||
|
console.log(partial)
|
||||||
|
}
|
||||||
|
console.log('end', feedback)
|
||||||
|
}
|
||||||
|
|
||||||
|
test()
|
@ -1,53 +1,53 @@
|
|||||||
import { parsePartialJson } from '@ai-sdk/ui-utils';
|
import { parsePartialJson } from '@ai-sdk/ui-utils'
|
||||||
import { z } from 'zod';
|
import { z } from 'zod'
|
||||||
|
|
||||||
export type DeepPartial<T> = T extends object
|
export type DeepPartial<T> = T extends object
|
||||||
? T extends Array<any>
|
? T extends Array<any>
|
||||||
? T
|
? T
|
||||||
: { [P in keyof T]?: DeepPartial<T[P]> }
|
: { [P in keyof T]?: DeepPartial<T[P]> }
|
||||||
: T;
|
: T
|
||||||
|
|
||||||
|
|
||||||
export function removeJsonMarkdown(text: string) {
|
export function removeJsonMarkdown(text: string) {
|
||||||
if (text.startsWith('```json')) {
|
if (text.startsWith('```json')) {
|
||||||
text = text.slice(7);
|
text = text.slice(7)
|
||||||
} else if (text.startsWith('json')) {
|
} else if (text.startsWith('json')) {
|
||||||
text = text.slice(4);
|
text = text.slice(4)
|
||||||
} else if (text.startsWith('```')) {
|
} else if (text.startsWith('```')) {
|
||||||
text = text.slice(3);
|
text = text.slice(3)
|
||||||
}
|
}
|
||||||
if (text.endsWith('```')) {
|
if (text.endsWith('```')) {
|
||||||
text = text.slice(0, -3);
|
text = text.slice(0, -3)
|
||||||
}
|
}
|
||||||
return text;
|
return text
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 解析流式的 JSON 数据
|
* 解析流式的 JSON 数据
|
||||||
* @param textStream 字符串流
|
* @param textStream 字符串流
|
||||||
* @param schema zod schema 用于类型验证
|
* @param _schema zod schema 用于类型验证
|
||||||
* @param isValid 自定义验证函数,用于判断解析出的 JSON 是否有效
|
* @param isValid 自定义验证函数,用于判断解析出的 JSON 是否有效
|
||||||
* @returns 异步生成器,yield 解析后的数据
|
* @returns 异步生成器,yield 解析后的数据
|
||||||
*/
|
*/
|
||||||
export async function* parseStreamingJson<T extends z.ZodType>(
|
export async function* parseStreamingJson<T extends z.ZodType>(
|
||||||
textStream: AsyncIterable<string>,
|
textStream: AsyncIterable<string>,
|
||||||
schema: T,
|
_schema: T,
|
||||||
isValid: (value: DeepPartial<z.infer<T>>) => boolean
|
isValid: (value: DeepPartial<z.infer<T>>) => boolean,
|
||||||
): AsyncGenerator<DeepPartial<z.infer<T>>> {
|
): AsyncGenerator<DeepPartial<z.infer<T>>> {
|
||||||
let rawText = '';
|
let rawText = ''
|
||||||
let isParseSuccessful = false;
|
let isParseSuccessful = false
|
||||||
|
|
||||||
for await (const chunk of textStream) {
|
for await (const chunk of textStream) {
|
||||||
rawText = removeJsonMarkdown(rawText + chunk);
|
rawText = removeJsonMarkdown(rawText + chunk)
|
||||||
const parsed = parsePartialJson(rawText);
|
const parsed = parsePartialJson(rawText)
|
||||||
|
|
||||||
isParseSuccessful = parsed.state === 'repaired-parse' || parsed.state === 'successful-parse';
|
isParseSuccessful =
|
||||||
if (isParseSuccessful) {
|
parsed.state === 'repaired-parse' || parsed.state === 'successful-parse'
|
||||||
yield parsed.value as DeepPartial<z.infer<T>>;
|
if (isParseSuccessful && isValid(parsed.value as any)) {
|
||||||
|
yield parsed.value as DeepPartial<z.infer<T>>
|
||||||
} else {
|
} else {
|
||||||
console.dir(parsed, { depth: null, colors: true });
|
console.dir(parsed, { depth: null, colors: true })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return { isSuccessful: isParseSuccessful };
|
return { isSuccessful: isParseSuccessful }
|
||||||
}
|
}
|
||||||
|
Reference in New Issue
Block a user