use std::fs::{self, File};
use std::io::{self, BufRead, BufReader, BufWriter, Write};
use std::path::Path;

pub fn merge_english_jsonl() -> io::Result<()> {
    let input_dir = "data/LLM_English";
    let output_dir = "output";
    let output_file = "english_master.jsonl";

    fs::create_dir_all(output_dir)?;

    let output_path = Path::new(output_dir).join(output_file);
    let mut writer = BufWriter::new(File::create(&output_path)?);

    let files = [
        "english_explain.jsonl",
        "english_reasoning.jsonl",
        "english_summary.jsonl",
        "instruction_following.jsonl",
    ];

    let mut total_lines: usize = 0;

    for filename in &files {
        let input_path = Path::new(input_dir).join(filename);

        if !input_path.exists() {
            println!("⚠️  Warning: {} not found, skipping...", filename);
            continue;
        }

        println!("📄 Merging: {}", filename);

        let file = File::open(&input_path)?;
        let reader = BufReader::new(file);

        for line in reader.lines() {
            let line = line?;
            let trimmed = line.trim();
            if !trimmed.is_empty() {
                writeln!(writer, "{}", trimmed)?;
                total_lines += 1;
            }
        }
    }

    writer.flush()?;
    println!("✅ Successfully merged {} lines into → {}", total_lines, output_path.display());
    Ok(())
}