init

lib/ai/providers.ts

@@ -0,0 +1,58 @@
+import { createOpenAI } from '@ai-sdk/openai';
+import { getEncoding } from 'js-tiktoken';
+
+import { RecursiveCharacterTextSplitter } from './text-splitter';
+
+// Providers
+const openai = createOpenAI({
+  apiKey: import.meta.env.VITE_OPENAI_API_KEY!,
+  baseURL: import.meta.env.VITE_OPENAI_ENDPOINT || 'https://api.openai.com/v1',
+});
+
+const customModel = import.meta.env.VITE_OPENAI_MODEL || 'o3-mini';
+
+// Models
+
+export const o3MiniModel = openai(customModel, {
+  // reasoningEffort: customModel.startsWith('o') ? 'medium' : undefined,
+  structuredOutputs: true,
+});
+
+const MinChunkSize = 140;
+const encoder = getEncoding('o200k_base');
+
+// trim prompt to maximum context size
+export function trimPrompt(
+  prompt: string,
+  contextSize = Number(import.meta.env.VITE_CONTEXT_SIZE) || 128_000,
+) {
+  if (!prompt) {
+    return '';
+  }
+
+  const length = encoder.encode(prompt).length;
+  if (length <= contextSize) {
+    return prompt;
+  }
+
+  const overflowTokens = length - contextSize;
+  // on average it's 3 characters per token, so multiply by 3 to get a rough estimate of the number of characters
+  const chunkSize = prompt.length - overflowTokens * 3;
+  if (chunkSize < MinChunkSize) {
+    return prompt.slice(0, MinChunkSize);
+  }
+
+  const splitter = new RecursiveCharacterTextSplitter({
+    chunkSize,
+    chunkOverlap: 0,
+  });
+  const trimmedPrompt = splitter.splitText(prompt)[0] ?? '';
+
+  // last catch, there's a chance that the trimmed prompt is same length as the original prompt, due to how tokens are split & innerworkings of the splitter, handle this case by just doing a hard cut
+  if (trimmedPrompt.length === prompt.length) {
+    return trimPrompt(prompt.slice(0, chunkSize), contextSize);
+  }
+
+  // recursively trim until the prompt is within the context size
+  return trimPrompt(trimmedPrompt, contextSize);
+}

lib/ai/text-splitter.test.ts

@@ -0,0 +1,77 @@
+import assert from 'node:assert';
+import { describe, it, beforeEach } from 'node:test';
+import { RecursiveCharacterTextSplitter } from './text-splitter';
+
+describe('RecursiveCharacterTextSplitter', () => {
+  let splitter: RecursiveCharacterTextSplitter;
+
+  beforeEach(() => {
+    splitter = new RecursiveCharacterTextSplitter({
+      chunkSize: 50,
+      chunkOverlap: 10,
+    });
+  });
+
+  it('Should correctly split text by separators', () => {
+    const text = 'Hello world, this is a test of the recursive text splitter.';
+
+    // Test with initial chunkSize
+    assert.deepEqual(
+      splitter.splitText(text),
+      ['Hello world', 'this is a test of the recursive text splitter']
+    );
+
+    // Test with updated chunkSize
+    splitter.chunkSize = 100;
+    assert.deepEqual(
+      splitter.splitText(
+        'Hello world, this is a test of the recursive text splitter. If I have a period, it should split along the period.'
+      ),
+      [
+        'Hello world, this is a test of the recursive text splitter',
+        'If I have a period, it should split along the period.',
+      ]
+    );
+
+    // Test with another updated chunkSize
+    splitter.chunkSize = 110;
+    assert.deepEqual(
+      splitter.splitText(
+        'Hello world, this is a test of the recursive text splitter. If I have a period, it should split along the period.\nOr, if there is a new line, it should prioritize splitting on new lines instead.'
+      ),
+      [
+        'Hello world, this is a test of the recursive text splitter',
+        'If I have a period, it should split along the period.',
+        'Or, if there is a new line, it should prioritize splitting on new lines instead.',
+      ]
+    );
+  });
+
+  it('Should handle empty string', () => {
+    assert.deepEqual(splitter.splitText(''), []);
+  });
+
+  it('Should handle special characters and large texts', () => {
+    const largeText = 'A'.repeat(1000);
+    splitter.chunkSize = 200;
+    assert.deepEqual(
+      splitter.splitText(largeText),
+      Array(5).fill('A'.repeat(200))
+    );
+
+    const specialCharText = 'Hello!@# world$%^ &*( this) is+ a-test';
+    assert.deepEqual(
+      splitter.splitText(specialCharText),
+      ['Hello!@#', 'world$%^', '&*( this)', 'is+', 'a-test']
+    );
+  });
+
+  it('Should handle chunkSize equal to chunkOverlap', () => {
+    splitter.chunkSize = 50;
+    splitter.chunkOverlap = 50;
+    assert.throws(
+      () => splitter.splitText('Invalid configuration'),
+      new Error('Cannot have chunkOverlap >= chunkSize')
+    );
+  });
+});

lib/ai/text-splitter.ts

@@ -0,0 +1,143 @@
+interface TextSplitterParams {
+  chunkSize: number;
+
+  chunkOverlap: number;
+}
+
+abstract class TextSplitter implements TextSplitterParams {
+  chunkSize = 1000;
+  chunkOverlap = 200;
+
+  constructor(fields?: Partial<TextSplitterParams>) {
+    this.chunkSize = fields?.chunkSize ?? this.chunkSize;
+    this.chunkOverlap = fields?.chunkOverlap ?? this.chunkOverlap;
+    if (this.chunkOverlap >= this.chunkSize) {
+      throw new Error('Cannot have chunkOverlap >= chunkSize');
+    }
+  }
+
+  abstract splitText(text: string): string[];
+
+  createDocuments(texts: string[]): string[] {
+    const documents: string[] = [];
+    for (let i = 0; i < texts.length; i += 1) {
+      const text = texts[i];
+      for (const chunk of this.splitText(text!)) {
+        documents.push(chunk);
+      }
+    }
+    return documents;
+  }
+
+  splitDocuments(documents: string[]): string[] {
+    return this.createDocuments(documents);
+  }
+
+  private joinDocs(docs: string[], separator: string): string | null {
+    const text = docs.join(separator).trim();
+    return text === '' ? null : text;
+  }
+
+  mergeSplits(splits: string[], separator: string): string[] {
+    const docs: string[] = [];
+    const currentDoc: string[] = [];
+    let total = 0;
+    for (const d of splits) {
+      const _len = d.length;
+      if (total + _len >= this.chunkSize) {
+        if (total > this.chunkSize) {
+          console.warn(
+            `Created a chunk of size ${total}, +
+which is longer than the specified ${this.chunkSize}`,
+          );
+        }
+        if (currentDoc.length > 0) {
+          const doc = this.joinDocs(currentDoc, separator);
+          if (doc !== null) {
+            docs.push(doc);
+          }
+          // Keep on popping if:
+          // - we have a larger chunk than in the chunk overlap
+          // - or if we still have any chunks and the length is long
+          while (
+            total > this.chunkOverlap ||
+            (total + _len > this.chunkSize && total > 0)
+          ) {
+            total -= currentDoc[0]!.length;
+            currentDoc.shift();
+          }
+        }
+      }
+      currentDoc.push(d);
+      total += _len;
+    }
+    const doc = this.joinDocs(currentDoc, separator);
+    if (doc !== null) {
+      docs.push(doc);
+    }
+    return docs;
+  }
+}
+
+export interface RecursiveCharacterTextSplitterParams
+  extends TextSplitterParams {
+  separators: string[];
+}
+
+export class RecursiveCharacterTextSplitter
+  extends TextSplitter
+  implements RecursiveCharacterTextSplitterParams
+{
+  separators: string[] = ['\n\n', '\n', '.', ',', '>', '<', ' ', ''];
+
+  constructor(fields?: Partial<RecursiveCharacterTextSplitterParams>) {
+    super(fields);
+    this.separators = fields?.separators ?? this.separators;
+  }
+
+  splitText(text: string): string[] {
+    const finalChunks: string[] = [];
+
+    // Get appropriate separator to use
+    let separator: string = this.separators[this.separators.length - 1]!;
+    for (const s of this.separators) {
+      if (s === '') {
+        separator = s;
+        break;
+      }
+      if (text.includes(s)) {
+        separator = s;
+        break;
+      }
+    }
+
+    // Now that we have the separator, split the text
+    let splits: string[];
+    if (separator) {
+      splits = text.split(separator);
+    } else {
+      splits = text.split('');
+    }
+
+    // Now go merging things, recursively splitting longer texts.
+    let goodSplits: string[] = [];
+    for (const s of splits) {
+      if (s.length < this.chunkSize) {
+        goodSplits.push(s);
+      } else {
+        if (goodSplits.length) {
+          const mergedText = this.mergeSplits(goodSplits, separator);
+          finalChunks.push(...mergedText);
+          goodSplits = [];
+        }
+        const otherInfo = this.splitText(s);
+        finalChunks.push(...otherInfo);
+      }
+    }
+    if (goodSplits.length) {
+      const mergedText = this.mergeSplits(goodSplits, separator);
+      finalChunks.push(...mergedText);
+    }
+    return finalChunks;
+  }
+}

lib/deep-research.ts

@@ -0,0 +1,303 @@
+import { generateObject, streamText } from 'ai';
+import { compact } from 'lodash-es';
+import pLimit from 'p-limit';
+import { z } from 'zod';
+import { parseStreamingJson, type DeepPartial } from '~/utils/json';
+
+import { o3MiniModel, trimPrompt } from './ai/providers';
+import { systemPrompt } from './prompt';
+import zodToJsonSchema from 'zod-to-json-schema';
+import { tavily, type TavilySearchResponse } from '@tavily/core';
+// import 'dotenv/config';
+
+// Used for streaming response
+type PartialSerpQueries = DeepPartial<z.infer<typeof serpQueriesTypeSchema>['queries']>;
+type PartialSearchResult = DeepPartial<z.infer<typeof serpResultTypeSchema>>;
+
+export type ResearchStep =
+  | { type: 'start'; message: string; depth: number; breadth: number }
+  | { type: 'generating_queries'; result: PartialSerpQueries; depth: number; breadth: number }
+  | { type: 'query_generated'; query: string; researchGoal: string; depth: number; breadth: number; queryIndex: number }
+  | { type: 'searching'; query: string; depth: number; breadth: number; queryIndex: number }
+  | { type: 'search_complete'; query: string; urls: string[]; depth: number; breadth: number; queryIndex: number }
+  | { type: 'processing_serach_result'; query: string; result: PartialSearchResult; depth: number; breadth: number; queryIndex: number }
+  | { type: 'error'; message: string }
+  | { type: 'complete' };
+
+// increase this if you have higher API rate limits
+const ConcurrencyLimit = 2;
+
+// Initialize Firecrawl with optional API key and optional base url
+
+// const firecrawl = new FirecrawlApp({
+//   apiKey: process.env.FIRECRAWL_KEY ?? '',
+//   apiUrl: process.env.FIRECRAWL_BASE_URL,
+// });
+const tvly = tavily({
+  apiKey: import.meta.env.VITE_TAVILY_API_KEY ?? '',
+})
+
+/**
+ * Schema for {@link generateSerpQueries} without dynamic descriptions
+ */
+export const serpQueriesTypeSchema = z.object({
+  queries: z.array(
+    z.object({
+      query: z.string(),
+      researchGoal: z.string(),
+    }),
+  ),
+});
+
+// take en user query, return a list of SERP queries
+export function generateSerpQueries({
+  query,
+  numQueries = 3,
+  learnings,
+}: {
+  query: string;
+  numQueries?: number;
+  // optional, if provided, the research will continue from the last learning
+  learnings?: string[];
+}) {
+  const schema = z.object({
+    queries: z
+      .array(
+        z.object({
+          query: z.string().describe('The SERP query'),
+          researchGoal: z
+            .string()
+            .describe(
+              'First talk about the goal of the research that this query is meant to accomplish, then go deeper into how to advance the research once the results are found, mention additional research directions. Be as specific as possible, especially for additional research directions.',
+            ),
+        }),
+      )
+      .describe(`List of SERP queries, max of ${numQueries}`)
+  })
+  const jsonSchema = JSON.stringify(zodToJsonSchema(schema));
+
+  const prompt = [
+    `Given the following prompt from the user, generate a list of SERP queries to research the topic. Return a maximum of ${numQueries} queries, but feel free to return less if the original prompt is clear. Make sure each query is unique and not similar to each other: <prompt>${query}</prompt>\n\n`,
+    learnings
+      ? `Here are some learnings from previous research, use them to generate more specific queries: ${learnings.join(
+        '\n',
+      )}`
+      : '',
+    `You MUST respond in JSON with the following schema: ${jsonSchema}`,
+  ].join('\n\n');
+  return streamText({
+    model: o3MiniModel,
+    system: systemPrompt(),
+    prompt,
+  });
+}
+
+export const serpResultTypeSchema = z.object({
+  learnings: z.array(z.string()),
+  followUpQuestions: z.array(z.string()),
+});
+function processSerpResult({
+  query,
+  result,
+  numLearnings = 3,
+  numFollowUpQuestions = 3,
+}: {
+  query: string;
+  // result: SearchResponse;
+  result: TavilySearchResponse
+  numLearnings?: number;
+  numFollowUpQuestions?: number;
+}) {
+  const schema = z.object({
+    learnings: z
+      .array(z.string())
+      .describe(`List of learnings, max of ${numLearnings}`),
+    followUpQuestions: z
+      .array(z.string())
+      .describe(
+        `List of follow-up questions to research the topic further, max of ${numFollowUpQuestions}`,
+      ),
+  });
+  const jsonSchema = JSON.stringify(zodToJsonSchema(schema));
+  const contents = compact(result.results.map(item => item.content)).map(
+    content => trimPrompt(content, 25_000),
+  );
+  const prompt = [
+    `Given the following contents from a SERP search for the query <query>${query}</query>, generate a list of learnings from the contents. Return a maximum of ${numLearnings} learnings, but feel free to return less if the contents are clear. Make sure each learning is unique and not similar to each other. The learnings should be concise and to the point, as detailed and information dense as possible. Make sure to include any entities like people, places, companies, products, things, etc in the learnings, as well as any exact metrics, numbers, or dates. The learnings will be used to research the topic further.`,
+    `<contents>${contents
+      .map(content => `<content>\n${content}\n</content>`)
+      .join('\n')}</contents>`,
+    `You MUST respond in JSON with the following schema: ${jsonSchema}`,
+  ].join('\n\n');
+
+  return streamText({
+    model: o3MiniModel,
+    abortSignal: AbortSignal.timeout(60_000),
+    system: systemPrompt(),
+    prompt,
+  });
+}
+
+export async function writeFinalReport({
+  prompt,
+  learnings,
+  visitedUrls,
+}: {
+  prompt: string;
+  learnings: string[];
+  visitedUrls: string[];
+}) {
+  const learningsString = trimPrompt(
+    learnings
+      .map(learning => `<learning>\n${learning}\n</learning>`)
+      .join('\n'),
+    150_000,
+  );
+
+  const res = await generateObject({
+    model: o3MiniModel,
+    system: systemPrompt(),
+    prompt: `Given the following prompt from the user, write a final report on the topic using the learnings from research. Make it as as detailed as possible, aim for 3 or more pages, include ALL the learnings from research:\n\n<prompt>${prompt}</prompt>\n\nHere are all the learnings from previous research:\n\n<learnings>\n${learningsString}\n</learnings>`,
+    schema: z.object({
+      reportMarkdown: z
+        .string()
+        .describe('Final report on the topic in Markdown'),
+    }),
+  });
+
+  // Append the visited URLs section to the report
+  const urlsSection = `\n\n## Sources\n\n${visitedUrls.map(url => `- ${url}`).join('\n')}`;
+  return res.object.reportMarkdown + urlsSection;
+}
+
+export async function deepResearch({
+  query,
+  breadth,
+  depth,
+  learnings = [],
+  visitedUrls = [],
+  onProgress,
+}: {
+  query: string;
+  breadth: number;
+  depth: number;
+  learnings?: string[];
+  visitedUrls?: string[];
+  onProgress: (step: ResearchStep) => void;
+}): Promise<void> {
+  onProgress({ type: 'start', message: `开始深度研究，深度：${depth}，广度：${breadth}`, depth, breadth });
+
+  try {
+    const serpQueriesResult = generateSerpQueries({
+      query,
+      learnings,
+      numQueries: breadth,
+    });
+    const limit = pLimit(ConcurrencyLimit);
+
+    let serpQueries: PartialSerpQueries = [];
+
+    for await (const parsedQueries of parseStreamingJson(
+      serpQueriesResult.textStream,
+      serpQueriesTypeSchema,
+      (value) => !!value.queries?.length && !!value.queries[0]?.query
+    )) {
+      if (parsedQueries.queries) {
+        serpQueries = parsedQueries.queries;
+        onProgress({
+          type: 'generating_queries',
+          result: serpQueries,
+          depth,
+          breadth
+        });
+      }
+    }
+
+    await Promise.all(
+      serpQueries.map(serpQuery =>
+        limit(async () => {
+          if (!serpQuery?.query) return
+          try {
+            // const result = await firecrawl.search(serpQuery.query, {
+            //   timeout: 15000,
+            //   limit: 5,
+            //   scrapeOptions: { formats: ['markdown'] },
+            // });
+            const result = await tvly.search(serpQuery.query, {
+              maxResults: 5,
+            })
+            console.log(`Ran ${serpQuery.query}, found ${result.results.length} contents`);
+
+            // Collect URLs from this search
+            const newUrls = compact(result.results.map(item => item.url));
+            const newBreadth = Math.ceil(breadth / 2);
+            const newDepth = depth - 1;
+
+            const serpResultGenerator = processSerpResult({
+              query: serpQuery.query,
+              result,
+              numFollowUpQuestions: newBreadth,
+            });
+            let serpResult: PartialSearchResult = {};
+
+            for await (const parsedLearnings of parseStreamingJson(
+              serpResultGenerator.textStream,
+              serpResultTypeSchema,
+              (value) => !!value.learnings?.length
+            )) {
+              serpResult = parsedLearnings;
+              onProgress({
+                type: 'processing_serach_result',
+                result: parsedLearnings,
+                depth,
+                breadth,
+                query: serpQuery.query,
+                queryIndex: serpQueries.indexOf(serpQuery),
+              });
+            }
+            console.log(`Processed serp result for ${serpQuery.query}`, serpResult);
+            const allLearnings = [...learnings, ...(serpResult.learnings ?? [])];
+            const allUrls = [...visitedUrls, ...newUrls];
+
+            if (newDepth > 0 && serpResult.followUpQuestions?.length) {
+              console.log(
+                `Researching deeper, breadth: ${newBreadth}, depth: ${newDepth}`,
+              );
+
+              const nextQuery = `
+              Previous research goal: ${serpQuery.researchGoal}
+              Follow-up research directions: ${serpResult.followUpQuestions.map(q => `\n${q}`).join('')}
+            `.trim();
+
+              return deepResearch({
+                query: nextQuery,
+                breadth: newBreadth,
+                depth: newDepth,
+                learnings: allLearnings,
+                visitedUrls: allUrls,
+                onProgress,
+              });
+            } else {
+              return {
+                learnings: allLearnings,
+                visitedUrls: allUrls,
+              };
+            }
+          } catch (e: any) {
+            throw new Error(`Error searching for ${serpQuery.query}, depth ${depth}\nMessage: ${e.message}`)
+          }
+        }),
+      ),
+    );
+  } catch (error: any) {
+    console.error(error);
+    onProgress({
+      type: 'error',
+      message: error?.message ?? 'Something went wrong',
+    })
+  }
+
+  onProgress({
+    type: 'complete',
+  });
+}

lib/feedback.ts

@@ -0,0 +1,37 @@
+import { streamText } from 'ai';
+import { z } from 'zod';
+import { zodToJsonSchema } from 'zod-to-json-schema'
+
+import { o3MiniModel } from './ai/providers';
+import { systemPrompt } from './prompt';
+
+export const feedbackTypeSchema = z.object({
+  questions: z.array(z.string())
+})
+
+export function generateFeedback({
+  query,
+  numQuestions = 3,
+}: {
+  query: string;
+  numQuestions?: number;
+}) {
+  const schema = z.object({
+    questions: z
+      .array(z.string())
+      .describe(
+        `Follow up questions to clarify the research direction, max of ${numQuestions}`,
+      ),
+  });
+  const jsonSchema = JSON.stringify(zodToJsonSchema(schema));
+  const prompt = [
+    `Given the following query from the user, ask some follow up questions to clarify the research direction. Return a maximum of ${numQuestions} questions, but feel free to return less if the original query is clear: <query>${query}</query>`,
+    `You MUST respond in JSON with the following schema: ${jsonSchema}`,
+  ].join('\n\n');
+  return streamText({
+    model: o3MiniModel,
+    system: systemPrompt(),
+    prompt,
+  });
+  // return userFeedback.object.questions.slice(0, numQuestions);
+}

lib/prompt.ts

@@ -0,0 +1,15 @@
+export const systemPrompt = () => {
+  const now = new Date().toISOString();
+  return `You are an expert researcher. Today is ${now}. Follow these instructions when responding:
+  - You may be asked to research subjects that is after your knowledge cutoff, assume the user is right when presented with news.
+  - The user is a highly experienced analyst, no need to simplify it, be as detailed as possible and make sure your response is correct.
+  - Be highly organized.
+  - Suggest solutions that I didn't think about.
+  - Be proactive and anticipate my needs.
+  - Treat me as an expert in all subject matter.
+  - Mistakes erode my trust, so be accurate and thorough.
+  - Provide detailed explanations, I'm comfortable with lots of detail.
+  - Value good arguments over authorities, the source is irrelevant.
+  - Consider new technologies and contrarian ideas, not just the conventional wisdom.
+  - You may use high levels of speculation or prediction, just flag it for me.`;
+};

lib/run.ts

@@ -0,0 +1,93 @@
+import * as fs from 'fs/promises';
+import * as readline from 'readline';
+
+import { deepResearch, writeFinalReport } from './deep-research';
+import { generateFeedback } from './feedback';
+
+const rl = readline.createInterface({
+  input: process.stdin,
+  output: process.stdout,
+});
+
+// Helper function to get user input
+function askQuestion(query: string): Promise<string> {
+  return new Promise(resolve => {
+    rl.question(query, answer => {
+      resolve(answer);
+    });
+  });
+}
+
+// run the agent
+async function run() {
+  // Get initial query
+  const initialQuery = await askQuestion('What would you like to research? ');
+
+  // Get breath and depth parameters
+  const breadth =
+    parseInt(
+      await askQuestion(
+        'Enter research breadth (recommended 2-10, default 4): ',
+      ),
+      10,
+    ) || 4;
+  const depth =
+    parseInt(
+      await askQuestion('Enter research depth (recommended 1-5, default 2): '),
+      10,
+    ) || 2;
+
+  console.log(`Creating research plan...`);
+
+  // Generate follow-up questions
+  const followUpQuestions = await generateFeedback({
+    query: initialQuery,
+  });
+
+  console.log(
+    '\nTo better understand your research needs, please answer these follow-up questions:',
+  );
+
+  // Collect answers to follow-up questions
+  const answers: string[] = [];
+  for (const question of followUpQuestions) {
+    const answer = await askQuestion(`\n${question}\nYour answer: `);
+    answers.push(answer);
+  }
+
+  // Combine all information for deep research
+  const combinedQuery = `
+Initial Query: ${initialQuery}
+Follow-up Questions and Answers:
+${followUpQuestions.map((q, i) => `Q: ${q}\nA: ${answers[i]}`).join('\n')}
+`;
+
+  console.log('\nResearching your topic...');
+
+  const { learnings, visitedUrls } = await deepResearch({
+    query: combinedQuery,
+    breadth,
+    depth,
+  });
+
+  console.log(`\n\nLearnings:\n\n${learnings.join('\n')}`);
+  console.log(
+    `\n\nVisited URLs (${visitedUrls.length}):\n\n${visitedUrls.join('\n')}`,
+  );
+  console.log('Writing final report...');
+
+  const report = await writeFinalReport({
+    prompt: combinedQuery,
+    learnings,
+    visitedUrls,
+  });
+
+  // Save report to file
+  await fs.writeFile('output.md', report, 'utf-8');
+
+  console.log(`\n\nFinal Report:\n\n${report}`);
+  console.log('\nReport has been saved to output.md');
+  rl.close();
+}
+
+run().catch(console.error);