feat: add support for Firecrawl

README.md

@@ -16,7 +16,7 @@ Features:
 Currently available providers:
 
 - AI: OpenAI compatible, DeepSeek, OpenRouter, Ollama
-- Web Search: Tavily (similar to Firecrawl, but with more free quota (1000 credits / month))
+- Web Search: Tavily (1000 free credits / month), Firecrawl
 
 Please give a 🌟 Star if you like this project!
 
@@ -26,7 +26,7 @@ Please give a 🌟 Star if you like this project!
 
 25/02/15
 
-- Added provider support for DeepSeek, OpenRouter and Ollama
+- Added AI providers DeepSeek, OpenRouter and Ollama; Added web search provider Firecrawl
 - Supported checking project updates
 - Supported regenerating reports
 - General fixes

README_zh.md

@@ -12,8 +12,7 @@
 
 当前支持的供应商：
 
-- AI 服务：OpenAPI 兼容、DeepSeek、OpenRouter、Ollama
+- AI 服务：OpenAPI 每月 1000 次免费搜索）、Firecrawl
-- 网络搜索：Tavily（类似 Firecrawl，提供每月 1000 次免费搜索）
 
 喜欢本项目请点 ⭐ 收藏！ <video width="500" src="https://github.com/user-attachments/assets/2f5a6f9c-18d1-4d40-9822-2de260d55dab" controls></video>
 
@@ -21,7 +20,7 @@
 
 25/02/15
 
-- AI 提供商支持 DeepSeek，OpenRouter 和 Ollama
+- AI 提供商支持 DeepSeek，OpenRouter 和 Ollama，联网搜素支持 Firecrawl
 - 支持检查项目更新
 - 支持重新生成报告
 - 一般性优化和改进

components/ConfigManager.vue

@@ -39,9 +39,32 @@
       value: 'ollama',
     },
   ])
+  const webSearchProviderOptions = computed(() => [
+    {
+      label: 'Tavily',
+      value: 'tavily',
+      help: 'settings.webSearch.providers.tavily.help',
+      // Only kept for easy reference in i18n Ally
+      _help: t('settings.webSearch.providers.tavily.help'),
+      link: 'https://app.tavily.com/home',
+    },
+    {
+      label: 'Firecrawl',
+      value: 'firecrawl',
+      help: 'settings.webSearch.providers.firecrawl.help',
+      // Only kept for easy reference in i18n Ally
+      _help: t('settings.webSearch.providers.firecrawl.help'),
+      link: 'https://www.firecrawl.dev/app/api-keys',
+    },
+  ])
   const selectedAiProvider = computed(() =>
     aiProviderOptions.value.find((o) => o.value === config.value.ai.provider),
   )
+  const selectedWebSearchProvider = computed(() =>
+    webSearchProviderOptions.value.find(
+      (o) => o.value === config.value.webSearch.provider,
+    ),
+  )
 
   // Try to find available AI models based on selected provider
   const debouncedListAiModels = useDebounceFn(async () => {
@@ -125,7 +148,7 @@
             </template>
             <USelect
               v-model="config.ai.provider"
-              class="w-50"
+              class="w-auto"
               :items="aiProviderOptions"
             />
           </UFormField>
@@ -174,20 +197,25 @@
           <h3 class="font-bold"> {{ $t('settings.webSearch.provider') }} </h3>
           <UFormField>
             <template #help>
-              <i18n-t keypath="settings.webSearch.providerHelp" tag="p">
+              <i18n-t
+                v-if="selectedWebSearchProvider?.help"
+                :keypath="selectedWebSearchProvider.help"
+                tag="p"
+              >
                 <UButton
                   class="!p-0"
-                  to="https://app.tavily.com/home"
+                  :to="selectedWebSearchProvider.link"
                   target="_blank"
                   variant="link"
                 >
-                  app.tavily.com
+                  {{ selectedWebSearchProvider.link }}
                 </UButton>
               </i18n-t>
             </template>
             <USelect
               v-model="config.webSearch.provider"
-              :items="[{ label: 'Tavily', value: 'tavily' }]"
+              class="w-auto"
+              :items="webSearchProviderOptions"
             />
           </UFormField>
           <UFormField :label="$t('settings.webSearch.apiKey')" required>

components/DeepResearch.vue

@@ -205,7 +205,7 @@
         query: getCombinedQuery(form.value, feedback.value),
         maxDepth: form.value.depth,
         breadth: form.value.breadth,
-        language: t('language', {}, { locale: locale.value }),
+        languageCode: locale.value,
         searchLanguage,
         onProgress: handleResearchProgress,
       })

composables/useTavily.ts

@@ -1,9 +0,0 @@
-import { tavily } from '@tavily/core'
-
-export const useTavily = () => {
-  const config = useConfigStore()
-  const tvly = tavily({
-    apiKey: config.config.webSearch.apiKey,
-  })
-  return tvly
-}

composables/useWebSearch.ts

@@ -0,0 +1,60 @@
+import { tavily } from '@tavily/core'
+import Firecrawl from '@mendable/firecrawl-js'
+
+type WebSearchOptions = {
+  maxResults?: number
+  /** The search language, e.g. `en`. Only works for Firecrawl. */
+  lang?: string
+}
+
+export type WebSearchResult = {
+  content: string
+  url: string
+  title?: string
+}
+
+type WebSearchFunction = (
+  query: string,
+  options: WebSearchOptions,
+) => Promise<WebSearchResult[]>
+
+export const useWebSearch = (): WebSearchFunction => {
+  const { config } = useConfigStore()
+
+  switch (config.webSearch.provider) {
+    case 'firecrawl': {
+      const fc = new Firecrawl({
+        apiKey: config.webSearch.apiKey,
+      })
+      return async (q: string, o: WebSearchOptions) => {
+        const results = await fc.search(q, o)
+        if (results.error) {
+          throw new Error(results.error)
+        }
+        return results.data
+          .filter((x) => !!x?.markdown && !!x.url)
+          .map((r) => ({
+            content: r.markdown!,
+            url: r.url!,
+            title: r.title,
+          }))
+      }
+    }
+    case 'tavily':
+    default: {
+      const tvly = tavily({
+        apiKey: config.webSearch.apiKey,
+      })
+      return async (q: string, o: WebSearchOptions) => {
+        const results = await tvly.search(q, o)
+        return results.results
+          .filter((x) => !!x?.content && !!x.url)
+          .map((r) => ({
+            content: r.content,
+            url: r.url,
+            title: r.title,
+          }))
+      }
+    }
+  }
+}

i18n/en.json

@@ -28,10 +28,17 @@
     },
     "webSearch": {
       "provider": "Web Search Provider",
-      "providerHelp": "Currently only supports Tavily. It provides lots of free quota (1000 credits / month).\nGet one API key at {0}.",
       "apiKey": "API Key",
       "queryLanguage": "Query Language",
-      "queryLanguageHelp": "The language of the search query. Useful if you want to get search results in a different language.\nWhen writing conclusions, the AI model will still use the language same as the web UI."
+      "queryLanguageHelp": "The language of the search query. Useful if you want to get search results in a different language.\nWhen writing conclusions, the AI model will still use the language same as the web UI.",
+      "providers": {
+        "tavily": {
+          "help": "Similar to Firecrawl, but provides 1000 free credits / month. Get one API key at {0}."
+        },
+        "firecrawl": {
+          "help": "Get one API key at {0}."
+        }
+      }
     }
   },
   "researchTopic": {

i18n/zh.json

@@ -28,12 +28,17 @@
     },
     "webSearch": {
       "provider": "联网搜索服务",
-      "providerHelp": "目前仅支持 Tavily，每个月可以免费搜索 1000 次。\n请在 {0} 生成一个 API 密钥。",
       "apiKey": "API 密钥",
       "queryLanguage": "使用语言",
       "queryLanguageHelp": "修改搜索词的语言。如果你想获取不同的搜索结果（比如查询高质量的英文资料），可以在这里修改。\nAI 模型在总结的时候仍然会使用当前网页的语言。",
-      "nodeFailed": "搜索失败",
+      "providers": {
-      "nodeFailedToast": "搜索步骤 “{label}” 失败"
+        "firecrawl": {
+          "help": "在 {0} 获取一个 API key。"
+        },
+        "tavily": {
+          "help": "和 Firecrawl 类似，不过提供了每月 1000 次免费搜索。在 {0} 获取一个 API key。"
+        }
+      }
     }
   },
   "researchTopic": {
@@ -69,7 +74,9 @@
     "researchGoal": "研究目标",
     "visitedUrls": "访问网址",
     "learnings": "结论",
-    "generating": "生成中..."
+    "generating": "生成中...",
+    "nodeFailed": "搜索失败",
+    "nodeFailedToast": "搜索步骤 “{label}” 失败"
   },
   "researchReport": {
     "title": "4. 研究报告",

lib/deep-research.ts

@@ -6,9 +6,8 @@ import { parseStreamingJson, type DeepPartial } from '~/utils/json'
 import { trimPrompt } from './ai/providers'
 import { languagePrompt, systemPrompt } from './prompt'
 import zodToJsonSchema from 'zod-to-json-schema'
-import { type TavilySearchResponse } from '@tavily/core'
-import { useTavily } from '~/composables/useTavily'
 import { useAiModel } from '~/composables/useAiProvider'
+import type { Locale } from '~/components/LangSwitcher.vue'
 
 export type ResearchResult = {
   learnings: string[]
@@ -141,7 +140,7 @@ function processSearchResult({
   language,
 }: {
   query: string
-  result: TavilySearchResponse
+  result: WebSearchResult[]
   language: string
   numLearnings?: number
   numFollowUpQuestions?: number
@@ -157,10 +156,7 @@ function processSearchResult({
       ),
   })
   const jsonSchema = JSON.stringify(zodToJsonSchema(schema))
-  const contents = result.results
+  const contents = result.map((item) => trimPrompt(item.content, 25_000))
-    .map((item) => item.content)
-    .filter(Boolean)
-    .map((content) => trimPrompt(content, 25_000))
   const prompt = [
     `Given the following contents from a SERP search for the query <query>${query}</query>, generate a list of learnings from the contents. Return a maximum of ${numLearnings} learnings, but feel free to return less if the contents are clear. Make sure each learning is unique and not similar to each other. The learnings should be concise and to the point, as detailed and information dense as possible. Make sure to include any entities like people, places, companies, products, things, etc in the learnings, as well as any exact metrics, numbers, or dates. The learnings will be used to research the topic further.`,
     `<contents>${contents
@@ -219,7 +215,7 @@ export async function deepResearch({
   query,
   breadth,
   maxDepth,
-  language,
+  languageCode,
   learnings = [],
   visitedUrls = [],
   onProgress,
@@ -230,7 +226,8 @@ export async function deepResearch({
   query: string
   breadth: number
   maxDepth: number
-  language: string
+  /** Language code */
+  languageCode: Locale
   learnings?: string[]
   visitedUrls?: string[]
   onProgress: (step: ResearchStep) => void
@@ -240,6 +237,7 @@ export async function deepResearch({
   searchLanguage?: string
 }): Promise<ResearchResult> {
   const { t } = useNuxtApp().$i18n
+  const language = t('language', {}, { locale: languageCode })
 
   try {
     const searchQueriesResult = generateSearchQueries({
@@ -321,17 +319,16 @@ export async function deepResearch({
           })
           try {
             // Use Tavily to search the web
-            const result = await useTavily().search(searchQuery.query, {
+            const result = await useWebSearch()(searchQuery.query, {
               maxResults: 5,
+              lang: languageCode,
             })
             console.log(
-              `Ran ${searchQuery.query}, found ${result.results.length} contents`,
+              `Ran ${searchQuery.query}, found ${result.length} contents`,
             )
 
             // Collect URLs from this search
-            const newUrls = result.results
+            const newUrls = result.map((item) => item.url).filter(Boolean)
-              .map((item) => item.url)
-              .filter(Boolean)
             onProgress({
               type: 'search_complete',
               urls: newUrls,
@@ -429,7 +426,7 @@ export async function deepResearch({
                 onProgress,
                 currentDepth: nextDepth,
                 nodeId: childNodeId(nodeId, i),
-                language,
+                languageCode,
               })
             } else {
               return {

package.json

@@ -17,6 +17,7 @@
     "@ai-sdk/ui-utils": "^1.1.14",
     "@ai-sdk/vue": "^1.1.17",
     "@iconify-json/lucide": "^1.2.26",
+    "@mendable/firecrawl-js": "^1.17.0",
     "@nuxt/ui": "3.0.0-alpha.12",
     "@nuxtjs/color-mode": "^3.5.2",
     "@nuxtjs/i18n": "9.2.0",

pnpm-lock.yaml

@@ -23,6 +23,9 @@ importers:
       '@iconify-json/lucide':
         specifier: ^1.2.26
         version: 1.2.26
+      '@mendable/firecrawl-js':
+        specifier: ^1.17.0
+        version: 1.17.0(ws@8.18.0)
       '@nuxt/ui':
         specifier: 3.0.0-alpha.12
         version: 3.0.0-alpha.12(@babel/parser@7.26.8)(axios@1.7.9)(change-case@5.4.4)(db0@0.2.3)(embla-carousel@8.5.2)(ioredis@5.5.0)(magicast@0.3.5)(radix-vue@1.9.13(vue@3.5.13(typescript@5.7.3)))(rollup@4.34.6)(typescript@5.7.3)(vite@6.1.0(@types/node@22.13.1)(jiti@2.4.2)(lightningcss@1.29.1)(terser@5.38.1)(yaml@2.7.0))(vue@3.5.13(typescript@5.7.3))
@@ -703,6 +706,9 @@ packages:
     engines: {node: '>=18'}
     hasBin: true
 
+  '@mendable/firecrawl-js@1.17.0':
+    resolution: {integrity: sha512-W8NEbFLtgedSI4CwxDFJ2iwcmwL7F3Gkv8usagYQ764AxnNdmcylVWMuYoQmzS6iYCtmSLFQUNEvHW9NbWigPQ==}
+
   '@miyaneee/rollup-plugin-json5@1.2.0':
     resolution: {integrity: sha512-JjTIaXZp9WzhUHpElrqPnl1AzBi/rvRs065F71+aTmlqvTMVkdbjZ8vfFl4nRlgJy+TPBw69ZK4pwFdmOAt4aA==}
     peerDependencies:
@@ -2562,6 +2568,11 @@ packages:
   isexe@2.0.0:
     resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==}
 
+  isows@1.0.6:
+    resolution: {integrity: sha512-lPHCayd40oW98/I0uvgaHKWCSvkzY27LjWLbtzOm64yQ+G3Q5npjjbdppU65iZXkK1Zt+kH9pfegli0AYfwYYw==}
+    peerDependencies:
+      ws: '*'
+
   jackspeak@3.4.3:
     resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==}
 
@@ -3818,6 +3829,9 @@ packages:
   type-level-regexp@0.1.17:
     resolution: {integrity: sha512-wTk4DH3cxwk196uGLK/E9pE45aLfeKJacKmcEgEOA/q5dnPGNxXt0cfYdFxb57L+sEpf1oJH4Dnx/pnRcku9jg==}
 
+  typescript-event-target@1.1.1:
+    resolution: {integrity: sha512-dFSOFBKV6uwaloBCCUhxlD3Pr/P1a/tJdcmPrTXCHlEFD3faj0mztjcGn6VBAhQ0/Bdy8K3VWrrqwbt/ffsYsg==}
+
   typescript@5.7.3:
     resolution: {integrity: sha512-84MVSjMEHP+FQRPy3pX9sTVV/INIex71s9TL2Gm5FG/WG1SqXeKyZ0k7/blY/4FdOzI12CBy1vGc4og/eus0fw==}
     engines: {node: '>=14.17'}
@@ -4940,6 +4954,17 @@ snapshots:
       - encoding
       - supports-color
 
+  '@mendable/firecrawl-js@1.17.0(ws@8.18.0)':
+    dependencies:
+      axios: 1.7.9
+      isows: 1.0.6(ws@8.18.0)
+      typescript-event-target: 1.1.1
+      zod: 3.24.2
+      zod-to-json-schema: 3.24.1(zod@3.24.2)
+    transitivePeerDependencies:
+      - debug
+      - ws
+
   '@miyaneee/rollup-plugin-json5@1.2.0(rollup@4.34.6)':
     dependencies:
       '@rollup/pluginutils': 5.1.4(rollup@4.34.6)
@@ -7188,6 +7213,10 @@ snapshots:
 
   isexe@2.0.0: {}
 
+  isows@1.0.6(ws@8.18.0):
+    dependencies:
+      ws: 8.18.0
+
   jackspeak@3.4.3:
     dependencies:
       '@isaacs/cliui': 8.0.2
@@ -8598,6 +8627,8 @@ snapshots:
 
   type-level-regexp@0.1.17: {}
 
+  typescript-event-target@1.1.1: {}
+
   typescript@5.7.3: {}
 
   ufo@1.5.4: {}

stores/config.ts

@@ -7,6 +7,8 @@ export type ConfigAiProvider =
   | 'deepseek'
   | 'ollama'
 
+export type ConfigWebSearchProvider = 'tavily' | 'firecrawl'
+
 export interface ConfigAi {
   provider: ConfigAiProvider
   apiKey?: string
@@ -15,7 +17,7 @@ export interface ConfigAi {
   contextSize?: number
 }
 export interface ConfigWebSearch {
-  provider: 'tavily'
+  provider: ConfigWebSearchProvider
   apiKey?: string
   /** Force the LLM to generate serp queries in a certain language */
   searchLanguage?: Locale