alavi_team_user2/Query-Doc-Generator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chaing max_length

Browse Source
This commit is contained in:
hediehloo 2025-12-10 07:54:19 +00:00
parent 71ec024a30
commit 10e01837e6
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/pipline.py
View File

@ -158,7 +158,7 @@ class Pipline:
def chunk_data(self, passage):
max_length = 3000
max_length = 8000
min_length = 30
if len(passage) < max_length:
@ -170,7 +170,7 @@ class Pipline:
start_idx = 0
stop_idx = 0
while True:
selected_lenth = random.choice([50, 100, 200, 300, 500, 800, 1300, 2000, 3000])
selected_lenth = random.choice([200, 400, 600, 800, 1200, 2000, 3200, 5200, 8000])
start_idx = stop_idx
one_passage, stop_idx = self.make_a_passage(selected_lenth, sentences, start_idx)