diff --git a/README.md b/README.md index 3d81251..a61df09 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Features: Currently available providers: - AI: OpenAI compatible, DeepSeek, OpenRouter, Ollama -- Web Search: Tavily (similar to Firecrawl, but with more free quota (1000 credits / month)) +- Web Search: Tavily (1000 free credits / month), Firecrawl Please give a 🌟 Star if you like this project! @@ -26,7 +26,7 @@ Please give a 🌟 Star if you like this project! 25/02/15 -- Added provider support for DeepSeek, OpenRouter and Ollama +- Added AI providers DeepSeek, OpenRouter and Ollama; Added web search provider Firecrawl - Supported checking project updates - Supported regenerating reports - General fixes diff --git a/README_zh.md b/README_zh.md index 7209108..b037ebf 100644 --- a/README_zh.md +++ b/README_zh.md @@ -12,8 +12,7 @@ 当前支持的供应商： -- AI 服务：OpenAPI 兼容、DeepSeek、OpenRouter、Ollama -- 网络搜索：Tavily（类似 Firecrawl，提供每月 1000 次免费搜索） +- AI 服务：OpenAPI 每月 1000 次免费搜索）、Firecrawl 喜欢本项目请点 ⭐ 收藏！ @@ -21,7 +20,7 @@ 25/02/15 -- AI 提供商支持 DeepSeek，OpenRouter 和 Ollama +- AI 提供商支持 DeepSeek，OpenRouter 和 Ollama，联网搜素支持 Firecrawl - 支持检查项目更新 - 支持重新生成报告 - 一般性优化和改进 diff --git a/components/ConfigManager.vue b/components/ConfigManager.vue index e27037e..8d821a5 100644 --- a/components/ConfigManager.vue +++ b/components/ConfigManager.vue @@ -39,9 +39,32 @@ value: 'ollama', }, ]) + const webSearchProviderOptions = computed(() => [ + { + label: 'Tavily', + value: 'tavily', + help: 'settings.webSearch.providers.tavily.help', + // Only kept for easy reference in i18n Ally + _help: t('settings.webSearch.providers.tavily.help'), + link: 'https://app.tavily.com/home', + }, + { + label: 'Firecrawl', + value: 'firecrawl', + help: 'settings.webSearch.providers.firecrawl.help', + // Only kept for easy reference in i18n Ally + _help: t('settings.webSearch.providers.firecrawl.help'), + link: 'https://www.firecrawl.dev/app/api-keys', + }, + ]) const selectedAiProvider = computed(() => aiProviderOptions.value.find((o) => o.value === config.value.ai.provider), ) + const selectedWebSearchProvider = computed(() => + webSearchProviderOptions.value.find( + (o) => o.value === config.value.webSearch.provider, + ), + ) // Try to find available AI models based on selected provider const debouncedListAiModels = useDebounceFn(async () => { @@ -125,7 +148,7 @@ @@ -174,20 +197,25 @@

{{ $t('settings.webSearch.provider') }}

diff --git a/components/DeepResearch.vue b/components/DeepResearch.vue index 4faa75e..f9dd9b4 100644 --- a/components/DeepResearch.vue +++ b/components/DeepResearch.vue @@ -205,7 +205,7 @@ query: getCombinedQuery(form.value, feedback.value), maxDepth: form.value.depth, breadth: form.value.breadth, - language: t('language', {}, { locale: locale.value }), + languageCode: locale.value, searchLanguage, onProgress: handleResearchProgress, }) diff --git a/composables/useTavily.ts b/composables/useTavily.ts deleted file mode 100644 index c41b26f..0000000 --- a/composables/useTavily.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { tavily } from '@tavily/core' - -export const useTavily = () => { - const config = useConfigStore() - const tvly = tavily({ - apiKey: config.config.webSearch.apiKey, - }) - return tvly -} diff --git a/composables/useWebSearch.ts b/composables/useWebSearch.ts new file mode 100644 index 0000000..1c30b28 --- /dev/null +++ b/composables/useWebSearch.ts @@ -0,0 +1,60 @@ +import { tavily } from '@tavily/core' +import Firecrawl from '@mendable/firecrawl-js' + +type WebSearchOptions = { + maxResults?: number + /** The search language, e.g. `en`. Only works for Firecrawl. */ + lang?: string +} + +export type WebSearchResult = { + content: string + url: string + title?: string +} + +type WebSearchFunction = ( + query: string, + options: WebSearchOptions, +) => Promise + +export const useWebSearch = (): WebSearchFunction => { + const { config } = useConfigStore() + + switch (config.webSearch.provider) { + case 'firecrawl': { + const fc = new Firecrawl({ + apiKey: config.webSearch.apiKey, + }) + return async (q: string, o: WebSearchOptions) => { + const results = await fc.search(q, o) + if (results.error) { + throw new Error(results.error) + } + return results.data + .filter((x) => !!x?.markdown && !!x.url) + .map((r) => ({ + content: r.markdown!, + url: r.url!, + title: r.title, + })) + } + } + case 'tavily': + default: { + const tvly = tavily({ + apiKey: config.webSearch.apiKey, + }) + return async (q: string, o: WebSearchOptions) => { + const results = await tvly.search(q, o) + return results.results + .filter((x) => !!x?.content && !!x.url) + .map((r) => ({ + content: r.content, + url: r.url, + title: r.title, + })) + } + } + } +} diff --git a/i18n/en.json b/i18n/en.json index 121e9b6..e622cfe 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -28,10 +28,17 @@ }, "webSearch": { "provider": "Web Search Provider", - "providerHelp": "Currently only supports Tavily. It provides lots of free quota (1000 credits / month).\nGet one API key at {0}.", "apiKey": "API Key", "queryLanguage": "Query Language", - "queryLanguageHelp": "The language of the search query. Useful if you want to get search results in a different language.\nWhen writing conclusions, the AI model will still use the language same as the web UI." + "queryLanguageHelp": "The language of the search query. Useful if you want to get search results in a different language.\nWhen writing conclusions, the AI model will still use the language same as the web UI.", + "providers": { + "tavily": { + "help": "Similar to Firecrawl, but provides 1000 free credits / month. Get one API key at {0}." + }, + "firecrawl": { + "help": "Get one API key at {0}." + } + } } }, "researchTopic": { @@ -92,4 +99,4 @@ "refresh": "Refresh page", "dismiss": "Dismiss" } -} \ No newline at end of file +} diff --git a/i18n/zh.json b/i18n/zh.json index d4e28d3..53b6808 100644 --- a/i18n/zh.json +++ b/i18n/zh.json @@ -28,12 +28,17 @@ }, "webSearch": { "provider": "联网搜索服务", - "providerHelp": "目前仅支持 Tavily，每个月可以免费搜索 1000 次。\n请在 {0} 生成一个 API 密钥。", "apiKey": "API 密钥", "queryLanguage": "使用语言", "queryLanguageHelp": "修改搜索词的语言。如果你想获取不同的搜索结果（比如查询高质量的英文资料），可以在这里修改。\nAI 模型在总结的时候仍然会使用当前网页的语言。", - "nodeFailed": "搜索失败", - "nodeFailedToast": "搜索步骤 “{label}” 失败" + "providers": { + "firecrawl": { + "help": "在 {0} 获取一个 API key。" + }, + "tavily": { + "help": "和 Firecrawl 类似，不过提供了每月 1000 次免费搜索。在 {0} 获取一个 API key。" + } + } } }, "researchTopic": { @@ -69,7 +74,9 @@ "researchGoal": "研究目标", "visitedUrls": "访问网址", "learnings": "结论", - "generating": "生成中..." + "generating": "生成中...", + "nodeFailed": "搜索失败", + "nodeFailedToast": "搜索步骤 “{label}” 失败" }, "researchReport": { "title": "4. 研究报告", @@ -92,4 +99,4 @@ "refresh": "刷新", "dismiss": "忽略" } -} \ No newline at end of file +} diff --git a/lib/deep-research.ts b/lib/deep-research.ts index 8f7c48b..94dbc50 100644 --- a/lib/deep-research.ts +++ b/lib/deep-research.ts @@ -6,9 +6,8 @@ import { parseStreamingJson, type DeepPartial } from '~/utils/json' import { trimPrompt } from './ai/providers' import { languagePrompt, systemPrompt } from './prompt' import zodToJsonSchema from 'zod-to-json-schema' -import { type TavilySearchResponse } from '@tavily/core' -import { useTavily } from '~/composables/useTavily' import { useAiModel } from '~/composables/useAiProvider' +import type { Locale } from '~/components/LangSwitcher.vue' export type ResearchResult = { learnings: string[] @@ -141,7 +140,7 @@ function processSearchResult({ language, }: { query: string - result: TavilySearchResponse + result: WebSearchResult[] language: string numLearnings?: number numFollowUpQuestions?: number @@ -157,10 +156,7 @@ function processSearchResult({ ), }) const jsonSchema = JSON.stringify(zodToJsonSchema(schema)) - const contents = result.results - .map((item) => item.content) - .filter(Boolean) - .map((content) => trimPrompt(content, 25_000)) + const contents = result.map((item) => trimPrompt(item.content, 25_000)) const prompt = [ `Given the following contents from a SERP search for the query ${query}, generate a list of learnings from the contents. Return a maximum of ${numLearnings} learnings, but feel free to return less if the contents are clear. Make sure each learning is unique and not similar to each other. The learnings should be concise and to the point, as detailed and information dense as possible. Make sure to include any entities like people, places, companies, products, things, etc in the learnings, as well as any exact metrics, numbers, or dates. The learnings will be used to research the topic further.`, `${contents @@ -219,7 +215,7 @@ export async function deepResearch({ query, breadth, maxDepth, - language, + languageCode, learnings = [], visitedUrls = [], onProgress, @@ -230,7 +226,8 @@ export async function deepResearch({ query: string breadth: number maxDepth: number - language: string + /** Language code */ + languageCode: Locale learnings?: string[] visitedUrls?: string[] onProgress: (step: ResearchStep) => void @@ -240,6 +237,7 @@ export async function deepResearch({ searchLanguage?: string }): Promise { const { t } = useNuxtApp().$i18n + const language = t('language', {}, { locale: languageCode }) try { const searchQueriesResult = generateSearchQueries({ @@ -321,17 +319,16 @@ export async function deepResearch({ }) try { // Use Tavily to search the web - const result = await useTavily().search(searchQuery.query, { + const result = await useWebSearch()(searchQuery.query, { maxResults: 5, + lang: languageCode, }) console.log( - `Ran ${searchQuery.query}, found ${result.results.length} contents`, + `Ran ${searchQuery.query}, found ${result.length} contents`, ) // Collect URLs from this search - const newUrls = result.results - .map((item) => item.url) - .filter(Boolean) + const newUrls = result.map((item) => item.url).filter(Boolean) onProgress({ type: 'search_complete', urls: newUrls, @@ -429,7 +426,7 @@ export async function deepResearch({ onProgress, currentDepth: nextDepth, nodeId: childNodeId(nodeId, i), - language, + languageCode, }) } else { return { diff --git a/package.json b/package.json index 6015dca..86603b5 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ "@ai-sdk/ui-utils": "^1.1.14", "@ai-sdk/vue": "^1.1.17", "@iconify-json/lucide": "^1.2.26", + "@mendable/firecrawl-js": "^1.17.0", "@nuxt/ui": "3.0.0-alpha.12", "@nuxtjs/color-mode": "^3.5.2", "@nuxtjs/i18n": "9.2.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e99f28a..86e04fe 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -23,6 +23,9 @@ importers: '@iconify-json/lucide': specifier: ^1.2.26 version: 1.2.26 + '@mendable/firecrawl-js': + specifier: ^1.17.0 + version: 1.17.0(ws@8.18.0) '@nuxt/ui': specifier: 3.0.0-alpha.12 version: 3.0.0-alpha.12(@babel/parser@7.26.8)(axios@1.7.9)(change-case@5.4.4)(db0@0.2.3)(embla-carousel@8.5.2)(ioredis@5.5.0)(magicast@0.3.5)(radix-vue@1.9.13(vue@3.5.13(typescript@5.7.3)))(rollup@4.34.6)(typescript@5.7.3)(vite@6.1.0(@types/node@22.13.1)(jiti@2.4.2)(lightningcss@1.29.1)(terser@5.38.1)(yaml@2.7.0))(vue@3.5.13(typescript@5.7.3)) @@ -703,6 +706,9 @@ packages: engines: {node: '>=18'} hasBin: true + '@mendable/firecrawl-js@1.17.0': + resolution: {integrity: sha512-W8NEbFLtgedSI4CwxDFJ2iwcmwL7F3Gkv8usagYQ764AxnNdmcylVWMuYoQmzS6iYCtmSLFQUNEvHW9NbWigPQ==} + '@miyaneee/rollup-plugin-json5@1.2.0': resolution: {integrity: sha512-JjTIaXZp9WzhUHpElrqPnl1AzBi/rvRs065F71+aTmlqvTMVkdbjZ8vfFl4nRlgJy+TPBw69ZK4pwFdmOAt4aA==} peerDependencies: @@ -2562,6 +2568,11 @@ packages: isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + isows@1.0.6: + resolution: {integrity: sha512-lPHCayd40oW98/I0uvgaHKWCSvkzY27LjWLbtzOm64yQ+G3Q5npjjbdppU65iZXkK1Zt+kH9pfegli0AYfwYYw==} + peerDependencies: + ws: '*' + jackspeak@3.4.3: resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==} @@ -3818,6 +3829,9 @@ packages: type-level-regexp@0.1.17: resolution: {integrity: sha512-wTk4DH3cxwk196uGLK/E9pE45aLfeKJacKmcEgEOA/q5dnPGNxXt0cfYdFxb57L+sEpf1oJH4Dnx/pnRcku9jg==} + typescript-event-target@1.1.1: + resolution: {integrity: sha512-dFSOFBKV6uwaloBCCUhxlD3Pr/P1a/tJdcmPrTXCHlEFD3faj0mztjcGn6VBAhQ0/Bdy8K3VWrrqwbt/ffsYsg==} + typescript@5.7.3: resolution: {integrity: sha512-84MVSjMEHP+FQRPy3pX9sTVV/INIex71s9TL2Gm5FG/WG1SqXeKyZ0k7/blY/4FdOzI12CBy1vGc4og/eus0fw==} engines: {node: '>=14.17'} @@ -4940,6 +4954,17 @@ snapshots: - encoding - supports-color + '@mendable/firecrawl-js@1.17.0(ws@8.18.0)': + dependencies: + axios: 1.7.9 + isows: 1.0.6(ws@8.18.0) + typescript-event-target: 1.1.1 + zod: 3.24.2 + zod-to-json-schema: 3.24.1(zod@3.24.2) + transitivePeerDependencies: + - debug + - ws + '@miyaneee/rollup-plugin-json5@1.2.0(rollup@4.34.6)': dependencies: '@rollup/pluginutils': 5.1.4(rollup@4.34.6) @@ -7188,6 +7213,10 @@ snapshots: isexe@2.0.0: {} + isows@1.0.6(ws@8.18.0): + dependencies: + ws: 8.18.0 + jackspeak@3.4.3: dependencies: '@isaacs/cliui': 8.0.2 @@ -8598,6 +8627,8 @@ snapshots: type-level-regexp@0.1.17: {} + typescript-event-target@1.1.1: {} + typescript@5.7.3: {} ufo@1.5.4: {} diff --git a/stores/config.ts b/stores/config.ts index 009431b..9fa70bc 100644 --- a/stores/config.ts +++ b/stores/config.ts @@ -7,6 +7,8 @@ export type ConfigAiProvider = | 'deepseek' | 'ollama' +export type ConfigWebSearchProvider = 'tavily' | 'firecrawl' + export interface ConfigAi { provider: ConfigAiProvider apiKey?: string @@ -15,7 +17,7 @@ export interface ConfigAi { contextSize?: number } export interface ConfigWebSearch { - provider: 'tavily' + provider: ConfigWebSearchProvider apiKey?: string /** Force the LLM to generate serp queries in a certain language */ searchLanguage?: Locale