jono/deep-research-web-ui
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

style: prettier format

Browse Source
This commit is contained in:
AnotiaWang
2025-02-11 22:57:49 +08:00
parent 84f63abb3d
commit d027965013
23 changed files with 485 additions and 331 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
lib/ai/text-splitter.test.ts
View File

@ -1,77 +1,80 @@
import assert from 'node:assert';
import { describe, it, beforeEach } from 'node:test';
import { RecursiveCharacterTextSplitter } from './text-splitter';
import assert from 'node:assert'
import { describe, it, beforeEach } from 'node:test'
import { RecursiveCharacterTextSplitter } from './text-splitter'
describe('RecursiveCharacterTextSplitter', () => {
let splitter: RecursiveCharacterTextSplitter;
let splitter: RecursiveCharacterTextSplitter
beforeEach(() => {
splitter = new RecursiveCharacterTextSplitter({
chunkSize: 50,
chunkOverlap: 10,
});
});
})
})
it('Should correctly split text by separators', () => {
const text = 'Hello world, this is a test of the recursive text splitter.';
const text = 'Hello world, this is a test of the recursive text splitter.'
// Test with initial chunkSize
assert.deepEqual(
splitter.splitText(text),
['Hello world', 'this is a test of the recursive text splitter']
);
assert.deepEqual(splitter.splitText(text), [
'Hello world',
'this is a test of the recursive text splitter',
])
// Test with updated chunkSize
splitter.chunkSize = 100;
splitter.chunkSize = 100
assert.deepEqual(
splitter.splitText(
'Hello world, this is a test of the recursive text splitter. If I have a period, it should split along the period.'
'Hello world, this is a test of the recursive text splitter. If I have a period, it should split along the period.',
),
[
'Hello world, this is a test of the recursive text splitter',
'If I have a period, it should split along the period.',
]
);
],
)
// Test with another updated chunkSize
splitter.chunkSize = 110;
splitter.chunkSize = 110
assert.deepEqual(
splitter.splitText(
'Hello world, this is a test of the recursive text splitter. If I have a period, it should split along the period.\nOr, if there is a new line, it should prioritize splitting on new lines instead.'
'Hello world, this is a test of the recursive text splitter. If I have a period, it should split along the period.\nOr, if there is a new line, it should prioritize splitting on new lines instead.',
),
[
'Hello world, this is a test of the recursive text splitter',
'If I have a period, it should split along the period.',
'Or, if there is a new line, it should prioritize splitting on new lines instead.',
]
);
});
],
)
})
it('Should handle empty string', () => {
assert.deepEqual(splitter.splitText(''), []);
});
assert.deepEqual(splitter.splitText(''), [])
})
it('Should handle special characters and large texts', () => {
const largeText = 'A'.repeat(1000);
splitter.chunkSize = 200;
const largeText = 'A'.repeat(1000)
splitter.chunkSize = 200
assert.deepEqual(
splitter.splitText(largeText),
Array(5).fill('A'.repeat(200))
);
Array(5).fill('A'.repeat(200)),
)
const specialCharText = 'Hello!@# world$%^ &*( this) is+ a-test';
assert.deepEqual(
splitter.splitText(specialCharText),
['Hello!@#', 'world$%^', '&*( this)', 'is+', 'a-test']
);
});
const specialCharText = 'Hello!@# world$%^ &*( this) is+ a-test'
assert.deepEqual(splitter.splitText(specialCharText), [
'Hello!@#',
'world$%^',
'&*( this)',
'is+',
'a-test',
])
})
it('Should handle chunkSize equal to chunkOverlap', () => {
splitter.chunkSize = 50;
splitter.chunkOverlap = 50;
splitter.chunkSize = 50
splitter.chunkOverlap = 50
assert.throws(
() => splitter.splitText('Invalid configuration'),
new Error('Cannot have chunkOverlap >= chunkSize')
);
});
});
new Error('Cannot have chunkOverlap >= chunkSize'),
)
})
})