Initial commit - AiRust cyber operator

src/converter.rs

@@ -0,0 +1,50 @@
+use std::fs::{self, File};
+use std::io::{self, BufRead, BufReader, BufWriter, Write};
+use std::path::Path;
+
+pub fn merge_english_jsonl() -> io::Result<()> {
+    let input_dir = "data/LLM_English";
+    let output_dir = "output";
+    let output_file = "english_master.jsonl";
+
+    fs::create_dir_all(output_dir)?;
+
+    let output_path = Path::new(output_dir).join(output_file);
+    let mut writer = BufWriter::new(File::create(&output_path)?);
+
+    let files = [
+        "english_explain.jsonl",
+        "english_reasoning.jsonl",
+        "english_summary.jsonl",
+        "instruction_following.jsonl",
+    ];
+
+    let mut total_lines: usize = 0;
+
+    for filename in &files {
+        let input_path = Path::new(input_dir).join(filename);
+
+        if !input_path.exists() {
+            println!("⚠️  Warning: {} not found, skipping...", filename);
+            continue;
+        }
+
+        println!("📄 Merging: {}", filename);
+
+        let file = File::open(&input_path)?;
+        let reader = BufReader::new(file);
+
+        for line in reader.lines() {
+            let line = line?;
+            let trimmed = line.trim();
+            if !trimmed.is_empty() {
+                writeln!(writer, "{}", trimmed)?;
+                total_lines += 1;
+            }
+        }
+    }
+
+    writer.flush()?;
+    println!("✅ Successfully merged {} lines into → {}", total_lines, output_path.display());
+    Ok(())
+}