frostyripper1
50 lines
1.4 KiB
Rust
50 lines
1.4 KiB
Rust
use std::fs::{self, File};
|
|
use std::io::{self, BufRead, BufReader, BufWriter, Write};
|
|
use std::path::Path;
|
|
|
|
pub fn merge_english_jsonl() -> io::Result<()> {
|
|
let input_dir = "data/LLM_English";
|
|
let output_dir = "output";
|
|
let output_file = "english_master.jsonl";
|
|
|
|
fs::create_dir_all(output_dir)?;
|
|
|
|
let output_path = Path::new(output_dir).join(output_file);
|
|
let mut writer = BufWriter::new(File::create(&output_path)?);
|
|
|
|
let files = [
|
|
"english_explain.jsonl",
|
|
"english_reasoning.jsonl",
|
|
"english_summary.jsonl",
|
|
"instruction_following.jsonl",
|
|
];
|
|
|
|
let mut total_lines: usize = 0;
|
|
|
|
for filename in &files {
|
|
let input_path = Path::new(input_dir).join(filename);
|
|
|
|
if !input_path.exists() {
|
|
println!("⚠️ Warning: {} not found, skipping...", filename);
|
|
continue;
|
|
}
|
|
|
|
println!("📄 Merging: {}", filename);
|
|
|
|
let file = File::open(&input_path)?;
|
|
let reader = BufReader::new(file);
|
|
|
|
for line in reader.lines() {
|
|
let line = line?;
|
|
let trimmed = line.trim();
|
|
if !trimmed.is_empty() {
|
|
writeln!(writer, "{}", trimmed)?;
|
|
total_lines += 1;
|
|
}
|
|
}
|
|
}
|
|
|
|
writer.flush()?;
|
|
println!("✅ Successfully merged {} lines into → {}", total_lines, output_path.display());
|
|
Ok(())
|
|
} |