jono/deep-research-web-ui
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add support for Firecrawl

Browse Source
This commit is contained in:
AnotiaWang
2025-02-15 21:41:15 +08:00
parent abb93139a6
commit f8af8b4afc
12 changed files with 167 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@ -16,7 +16,7 @@ Features:
Currently available providers:
- AI: OpenAI compatible, DeepSeek, OpenRouter, Ollama
- Web Search: Tavily (similar to Firecrawl, but with more free quota (1000 credits / month))
- Web Search: Tavily (1000 free credits / month), Firecrawl
Please give a 🌟 Star if you like this project!
@ -26,7 +26,7 @@ Please give a 🌟 Star if you like this project!
25/02/15
- Added provider support for DeepSeek, OpenRouter and Ollama
- Added AI providers DeepSeek, OpenRouter and Ollama; Added web search provider Firecrawl
- Supported checking project updates
- Supported regenerating reports
- General fixes

5
README_zh.md
View File

@ -12,8 +12,7 @@
当前支持的供应商：
- AI 服务：OpenAPI 兼容、DeepSeek、OpenRouter、Ollama
- 网络搜索：Tavily（类似 Firecrawl，提供每月 1000 次免费搜索）
- AI 服务：OpenAPI 每月 1000 次免费搜索）、Firecrawl
喜欢本项目请点 ⭐ 收藏！ <video width="500" src="https://github.com/user-attachments/assets/2f5a6f9c-18d1-4d40-9822-2de260d55dab" controls></video>
@ -21,7 +20,7 @@
25/02/15
- AI 提供商支持 DeepSeek，OpenRouter 和 Ollama
- AI 提供商支持 DeepSeek，OpenRouter 和 Ollama，联网搜素支持 Firecrawl
- 支持检查项目更新
- 支持重新生成报告
- 一般性优化和改进

38
components/ConfigManager.vue
View File

@ -39,9 +39,32 @@
value: 'ollama',
},
])
const webSearchProviderOptions = computed(() => [
{
label: 'Tavily',
value: 'tavily',
help: 'settings.webSearch.providers.tavily.help',
// Only kept for easy reference in i18n Ally
_help: t('settings.webSearch.providers.tavily.help'),
link: 'https://app.tavily.com/home',
},
{
label: 'Firecrawl',
value: 'firecrawl',
help: 'settings.webSearch.providers.firecrawl.help',
// Only kept for easy reference in i18n Ally
_help: t('settings.webSearch.providers.firecrawl.help'),
link: 'https://www.firecrawl.dev/app/api-keys',
},
])
const selectedAiProvider = computed(() =>
aiProviderOptions.value.find((o) => o.value === config.value.ai.provider),
)
const selectedWebSearchProvider = computed(() =>
webSearchProviderOptions.value.find(
(o) => o.value === config.value.webSearch.provider,
),
)
// Try to find available AI models based on selected provider
const debouncedListAiModels = useDebounceFn(async () => {
@ -125,7 +148,7 @@
</template>
<USelect
v-model="config.ai.provider"
class="w-50"
class="w-auto"
:items="aiProviderOptions"
/>
</UFormField>
@ -174,20 +197,25 @@
<h3 class="font-bold"> {{ $t('settings.webSearch.provider') }} </h3>
<UFormField>
<template #help>
<i18n-t keypath="settings.webSearch.providerHelp" tag="p">
<i18n-t
v-if="selectedWebSearchProvider?.help"
:keypath="selectedWebSearchProvider.help"
tag="p"
>
<UButton
class="!p-0"
to="https://app.tavily.com/home"
:to="selectedWebSearchProvider.link"
target="_blank"
variant="link"
>
app.tavily.com
{{ selectedWebSearchProvider.link }}
</UButton>
</i18n-t>
</template>
<USelect
v-model="config.webSearch.provider"
:items="[{ label: 'Tavily', value: 'tavily' }]"
class="w-auto"
:items="webSearchProviderOptions"
/>
</UFormField>
<UFormField :label="$t('settings.webSearch.apiKey')" required>

2
components/DeepResearch.vue
View File

@ -205,7 +205,7 @@
query: getCombinedQuery(form.value, feedback.value),
maxDepth: form.value.depth,
breadth: form.value.breadth,
language: t('language', {}, { locale: locale.value }),
languageCode: locale.value,
searchLanguage,
onProgress: handleResearchProgress,
})

9
composables/useTavily.ts
View File

@ -1,9 +0,0 @@
import { tavily } from '@tavily/core'
export const useTavily = () => {
const config = useConfigStore()
const tvly = tavily({
apiKey: config.config.webSearch.apiKey,
})
return tvly
}

60
composables/useWebSearch.ts Normal file
View File

@ -0,0 +1,60 @@
import { tavily } from '@tavily/core'
import Firecrawl from '@mendable/firecrawl-js'
type WebSearchOptions = {
maxResults?: number
/** The search language, e.g. `en`. Only works for Firecrawl. */
lang?: string
}
export type WebSearchResult = {
content: string
url: string
title?: string
}
type WebSearchFunction = (
query: string,
options: WebSearchOptions,
) => Promise<WebSearchResult[]>
export const useWebSearch = (): WebSearchFunction => {
const { config } = useConfigStore()
switch (config.webSearch.provider) {
case 'firecrawl': {
const fc = new Firecrawl({
apiKey: config.webSearch.apiKey,
})
return async (q: string, o: WebSearchOptions) => {
const results = await fc.search(q, o)
if (results.error) {
throw new Error(results.error)
}
return results.data
.filter((x) => !!x?.markdown && !!x.url)
.map((r) => ({
content: r.markdown!,
url: r.url!,
title: r.title,
}))
}
}
case 'tavily':
default: {
const tvly = tavily({
apiKey: config.webSearch.apiKey,
})
return async (q: string, o: WebSearchOptions) => {
const results = await tvly.search(q, o)
return results.results
.filter((x) => !!x?.content && !!x.url)
.map((r) => ({
content: r.content,
url: r.url,
title: r.title,
}))
}
}
}
}

11
i18n/en.json
View File

@ -28,10 +28,17 @@
},
"webSearch": {
"provider": "Web Search Provider",
"providerHelp": "Currently only supports Tavily. It provides lots of free quota (1000 credits / month).\nGet one API key at {0}.",
"apiKey": "API Key",
"queryLanguage": "Query Language",
"queryLanguageHelp": "The language of the search query. Useful if you want to get search results in a different language.\nWhen writing conclusions, the AI model will still use the language same as the web UI."
"queryLanguageHelp": "The language of the search query. Useful if you want to get search results in a different language.\nWhen writing conclusions, the AI model will still use the language same as the web UI.",
"providers": {
"tavily": {
"help": "Similar to Firecrawl, but provides 1000 free credits / month. Get one API key at {0}."
},
"firecrawl": {
"help": "Get one API key at {0}."
}
}
}
},
"researchTopic": {

15
i18n/zh.json
View File

@ -28,12 +28,17 @@
},
"webSearch": {
"provider": "联网搜索服务",
"providerHelp": "目前仅支持 Tavily，每个月可以免费搜索 1000 次。\n请在 {0} 生成一个 API 密钥。",
"apiKey": "API 密钥",
"queryLanguage": "使用语言",
"queryLanguageHelp": "修改搜索词的语言。如果你想获取不同的搜索结果（比如查询高质量的英文资料），可以在这里修改。\nAI 模型在总结的时候仍然会使用当前网页的语言。",
"nodeFailed": "搜索失败",
"nodeFailedToast": "搜索步骤 “{label}” 失败"
"providers": {
"firecrawl": {
"help": "在 {0} 获取一个 API key。"
},
"tavily": {
"help": "和 Firecrawl 类似，不过提供了每月 1000 次免费搜索。在 {0} 获取一个 API key。"
}
}
}
},
"researchTopic": {
@ -69,7 +74,9 @@
"researchGoal": "研究目标",
"visitedUrls": "访问网址",
"learnings": "结论",
"generating": "生成中..."
"generating": "生成中...",
"nodeFailed": "搜索失败",
"nodeFailedToast": "搜索步骤 “{label}” 失败"
},
"researchReport": {
"title": "4. 研究报告",

27
lib/deep-research.ts
View File

@ -6,9 +6,8 @@ import { parseStreamingJson, type DeepPartial } from '~/utils/json'
import { trimPrompt } from './ai/providers'
import { languagePrompt, systemPrompt } from './prompt'
import zodToJsonSchema from 'zod-to-json-schema'
import { type TavilySearchResponse } from '@tavily/core'
import { useTavily } from '~/composables/useTavily'
import { useAiModel } from '~/composables/useAiProvider'
import type { Locale } from '~/components/LangSwitcher.vue'
export type ResearchResult = {
learnings: string[]
@ -141,7 +140,7 @@ function processSearchResult({
language,
}: {
query: string
result: TavilySearchResponse
result: WebSearchResult[]
language: string
numLearnings?: number
numFollowUpQuestions?: number
@ -157,10 +156,7 @@ function processSearchResult({
),
})
const jsonSchema = JSON.stringify(zodToJsonSchema(schema))
const contents = result.results
.map((item) => item.content)
.filter(Boolean)
.map((content) => trimPrompt(content, 25_000))
const contents = result.map((item) => trimPrompt(item.content, 25_000))
const prompt = [
`Given the following contents from a SERP search for the query <query>${query}</query>, generate a list of learnings from the contents. Return a maximum of ${numLearnings} learnings, but feel free to return less if the contents are clear. Make sure each learning is unique and not similar to each other. The learnings should be concise and to the point, as detailed and information dense as possible. Make sure to include any entities like people, places, companies, products, things, etc in the learnings, as well as any exact metrics, numbers, or dates. The learnings will be used to research the topic further.`,
`<contents>${contents
@ -219,7 +215,7 @@ export async function deepResearch({
query,
breadth,
maxDepth,
language,
languageCode,
learnings = [],
visitedUrls = [],
onProgress,
@ -230,7 +226,8 @@ export async function deepResearch({
query: string
breadth: number
maxDepth: number
language: string
/** Language code */
languageCode: Locale
learnings?: string[]
visitedUrls?: string[]
onProgress: (step: ResearchStep) => void
@ -240,6 +237,7 @@ export async function deepResearch({
searchLanguage?: string
}): Promise<ResearchResult> {
const { t } = useNuxtApp().$i18n
const language = t('language', {}, { locale: languageCode })
try {
const searchQueriesResult = generateSearchQueries({
@ -321,17 +319,16 @@ export async function deepResearch({
})
try {
// Use Tavily to search the web
const result = await useTavily().search(searchQuery.query, {
const result = await useWebSearch()(searchQuery.query, {
maxResults: 5,
lang: languageCode,
})
console.log(
`Ran ${searchQuery.query}, found ${result.results.length} contents`,
`Ran ${searchQuery.query}, found ${result.length} contents`,
)
// Collect URLs from this search
const newUrls = result.results
.map((item) => item.url)
.filter(Boolean)
const newUrls = result.map((item) => item.url).filter(Boolean)
onProgress({
type: 'search_complete',
urls: newUrls,
@ -429,7 +426,7 @@ export async function deepResearch({
onProgress,
currentDepth: nextDepth,
nodeId: childNodeId(nodeId, i),
language,
languageCode,
})
} else {
return {

1
package.json
View File

@ -17,6 +17,7 @@
"@ai-sdk/ui-utils": "^1.1.14",
"@ai-sdk/vue": "^1.1.17",
"@iconify-json/lucide": "^1.2.26",
"@mendable/firecrawl-js": "^1.17.0",
"@nuxt/ui": "3.0.0-alpha.12",
"@nuxtjs/color-mode": "^3.5.2",
"@nuxtjs/i18n": "9.2.0",

31
pnpm-lock.yaml generated
View File

@ -23,6 +23,9 @@ importers:
'@iconify-json/lucide':
specifier: ^1.2.26
version: 1.2.26
'@mendable/firecrawl-js':
specifier: ^1.17.0
version: 1.17.0(ws@8.18.0)
'@nuxt/ui':
specifier: 3.0.0-alpha.12
version: 3.0.0-alpha.12(@babel/parser@7.26.8)(axios@1.7.9)(change-case@5.4.4)(db0@0.2.3)(embla-carousel@8.5.2)(ioredis@5.5.0)(magicast@0.3.5)(radix-vue@1.9.13(vue@3.5.13(typescript@5.7.3)))(rollup@4.34.6)(typescript@5.7.3)(vite@6.1.0(@types/node@22.13.1)(jiti@2.4.2)(lightningcss@1.29.1)(terser@5.38.1)(yaml@2.7.0))(vue@3.5.13(typescript@5.7.3))
@ -703,6 +706,9 @@ packages:
engines: {node: '>=18'}
hasBin: true
'@mendable/firecrawl-js@1.17.0':
resolution: {integrity: sha512-W8NEbFLtgedSI4CwxDFJ2iwcmwL7F3Gkv8usagYQ764AxnNdmcylVWMuYoQmzS6iYCtmSLFQUNEvHW9NbWigPQ==}
'@miyaneee/rollup-plugin-json5@1.2.0':
resolution: {integrity: sha512-JjTIaXZp9WzhUHpElrqPnl1AzBi/rvRs065F71+aTmlqvTMVkdbjZ8vfFl4nRlgJy+TPBw69ZK4pwFdmOAt4aA==}
peerDependencies:
@ -2562,6 +2568,11 @@ packages:
isexe@2.0.0:
resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==}
isows@1.0.6:
resolution: {integrity: sha512-lPHCayd40oW98/I0uvgaHKWCSvkzY27LjWLbtzOm64yQ+G3Q5npjjbdppU65iZXkK1Zt+kH9pfegli0AYfwYYw==}
peerDependencies:
ws: '*'
jackspeak@3.4.3:
resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==}
@ -3818,6 +3829,9 @@ packages:
type-level-regexp@0.1.17:
resolution: {integrity: sha512-wTk4DH3cxwk196uGLK/E9pE45aLfeKJacKmcEgEOA/q5dnPGNxXt0cfYdFxb57L+sEpf1oJH4Dnx/pnRcku9jg==}
typescript-event-target@1.1.1:
resolution: {integrity: sha512-dFSOFBKV6uwaloBCCUhxlD3Pr/P1a/tJdcmPrTXCHlEFD3faj0mztjcGn6VBAhQ0/Bdy8K3VWrrqwbt/ffsYsg==}
typescript@5.7.3:
resolution: {integrity: sha512-84MVSjMEHP+FQRPy3pX9sTVV/INIex71s9TL2Gm5FG/WG1SqXeKyZ0k7/blY/4FdOzI12CBy1vGc4og/eus0fw==}
engines: {node: '>=14.17'}
@ -4940,6 +4954,17 @@ snapshots:
- encoding
- supports-color
'@mendable/firecrawl-js@1.17.0(ws@8.18.0)':
dependencies:
axios: 1.7.9
isows: 1.0.6(ws@8.18.0)
typescript-event-target: 1.1.1
zod: 3.24.2
zod-to-json-schema: 3.24.1(zod@3.24.2)
transitivePeerDependencies:
- debug
- ws
'@miyaneee/rollup-plugin-json5@1.2.0(rollup@4.34.6)':
dependencies:
'@rollup/pluginutils': 5.1.4(rollup@4.34.6)
@ -7188,6 +7213,10 @@ snapshots:
isexe@2.0.0: {}
isows@1.0.6(ws@8.18.0):
dependencies:
ws: 8.18.0
jackspeak@3.4.3:
dependencies:
'@isaacs/cliui': 8.0.2
@ -8598,6 +8627,8 @@ snapshots:
type-level-regexp@0.1.17: {}
typescript-event-target@1.1.1: {}
typescript@5.7.3: {}
ufo@1.5.4: {}

4
stores/config.ts
View File

@ -7,6 +7,8 @@ export type ConfigAiProvider =
| 'deepseek'
| 'ollama'
export type ConfigWebSearchProvider = 'tavily' | 'firecrawl'
export interface ConfigAi {
provider: ConfigAiProvider
apiKey?: string
@ -15,7 +17,7 @@ export interface ConfigAi {
contextSize?: number
}
export interface ConfigWebSearch {
provider: 'tavily'
provider: ConfigWebSearchProvider
apiKey?: string
/** Force the LLM to generate serp queries in a certain language */
searchLanguage?: Locale