maybe done with recording

Cargo.lock

@@ -134,6 +134,12 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
+
 [[package]]
 name = "dump"
 version = "0.1.0"
@@ -205,6 +211,7 @@ version = "0.1.0"
 dependencies = [
  "ffmpeg-next",
  "retro-rs 0.5.4",
+ "ringbuf",
  "rply-codec",
 ]
 
@@ -390,6 +397,21 @@ version = "0.3.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
 
+[[package]]
+name = "portable-atomic"
+version = "1.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483"
+
+[[package]]
+name = "portable-atomic-util"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507"
+dependencies = [
+ "portable-atomic",
+]
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.103"
@@ -470,6 +492,17 @@ dependencies = [
  "rust-libretro-sys",
 ]
 
+[[package]]
+name = "ringbuf"
+version = "0.4.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fe47b720588c8702e34b5979cb3271a8b1842c7cb6f57408efa70c779363488c"
+dependencies = [
+ "crossbeam-utils",
+ "portable-atomic",
+ "portable-atomic-util",
+]
+
 [[package]]
 name = "rmp"
 version = "0.8.14"

genvideo/Cargo.toml

@@ -7,3 +7,4 @@ edition = "2024"
 rply-codec = { path = "../codec" }
 retro-rs = { version = "0.5.4", default-features=false, path = "../../retro-rs" }
 ffmpeg-next = "8.0.0"
+ringbuf = "0.4.8"

genvideo/src/main.rs

@@ -1,42 +1,27 @@
 use ffmpeg_next::{
-    codec::Debug,
+    software::converter as img_conv,
     util::frame::{Audio as FFAFrame, Video as FFVFrame},
 };
 use retro_rs::Emulator;
+use ringbuf::traits::{Consumer, Observer, RingBuffer};
 use rply_codec::{Frame, decode};
 use std::path::Path;
 
 fn copy_audio(samples: &[i16], frame: &mut FFAFrame) {
-    println!(
-        "Copy {} samples into {}",
-        samples.len() / 2,
-        frame.samples()
-    );
     for (i, pair) in samples.chunks_exact(2).enumerate() {
-        let [l, r] = pair else {
+        let [l, r] = pair else { unreachable!() };
-            unreachable!();
+        frame.plane_mut(0)[i] = *l as f32 / i16::MAX as f32;
-        };
+        frame.plane_mut(1)[i] = *r as f32 / i16::MAX as f32;
-        frame.plane_mut(0)[i] = f32::from(*l) / 65535.0;
-        frame.plane_mut(1)[i] = f32::from(*r) / 65535.0;
     }
 }
-fn copy_video(fb: &[u8], frame: &mut FFVFrame) {
+fn copy_video(
-    let w = i32::try_from(frame.width()).unwrap();
+    fb: &[u8],
-    let h = i32::try_from(frame.height()).unwrap();
+    converter: &mut ffmpeg_next::software::scaling::Context,
-    let fmt: ffmpeg_next::ffi::AVPixelFormat = frame.format().into();
+    rgbframe: &mut FFVFrame,
-    unsafe {
+    outframe: &mut FFVFrame,
-        let frame_ptr = frame.as_mut_ptr();
+) {
-        let ret = ffmpeg_next::ffi::av_image_fill_arrays(
+    rgbframe.data_mut(0).copy_from_slice(fb);
-            (*frame_ptr).data.as_mut_ptr(),
+    converter.run(rgbframe, outframe).unwrap();
-            (*frame_ptr).linesize.as_mut_ptr(),
-            fb.as_ptr(),
-            fmt,
-            w,
-            h,
-            1,
-        );
-        assert!(ret > 0, "av_image_fill_arrays returned {ret}");
-    }
 }
 
 fn main() {
@@ -69,50 +54,51 @@ fn main() {
     let emu_time_base = ffmpeg_next::util::rational::Rational::new(1, emu.get_video_fps() as i32);
     let out_video_codec = ffmpeg_next::encoder::find(ffmpeg_next::codec::Id::H264).unwrap();
     let mut out_video_ctx = ffmpeg_next::codec::context::Context::new_with_codec(out_video_codec);
+    // out_video_ctx.set_time_base(emu_time_base);
     let mut video_params = ffmpeg_next::codec::Parameters::new();
     unsafe {
         let vps = video_params.as_mut_ptr();
         (*vps).width = i32::try_from(w).unwrap();
         (*vps).height = i32::try_from(h).unwrap();
         (*vps).codec_id = out_video_codec.id().into();
-        (*vps).framerate = ffmpeg_next::util::rational::Rational::from(emu.get_video_fps()).into();
+        (*vps).codec_type = ffmpeg_next::ffi::AVMediaType::AVMEDIA_TYPE_VIDEO;
     };
     out_video_ctx.set_parameters(video_params).unwrap();
-    out_video_ctx.set_time_base(emu_time_base);
-    out_video_ctx.debug(Debug::all());
-    dbg!(out_video_ctx.codec().unwrap().id());
     let mut out_video = output.add_stream_with(&out_video_ctx).unwrap();
-    out_video.set_time_base(emu_time_base);
+    let mut encoded_video = ffmpeg_next::Packet::empty();
-    let video_time_base = out_video.time_base();
+    // out_video.set_time_base(emu_time_base);
     let mut out_video_enc = out_video_ctx.encoder().video().unwrap();
-    out_video_enc.debug(Debug::all());
     out_video_enc.set_format(ffmpeg_next::format::Pixel::YUV420P);
     out_video_enc.set_width(u32::try_from(w).unwrap());
     out_video_enc.set_height(u32::try_from(h).unwrap());
-    out_video_enc.set_time_base(video_time_base);
+    out_video_enc.set_time_base(emu_time_base);
     let mut out_video_enc = out_video_enc.open().unwrap();
     let out_audio_codec = ffmpeg_next::encoder::find(ffmpeg_next::codec::Id::AAC).unwrap();
     let mut out_audio_ctx = ffmpeg_next::codec::context::Context::new_with_codec(out_audio_codec);
-    out_audio_ctx.debug(Debug::all());
+    // out_audio_ctx.debug(Debug::all());
     let mut audio_params = ffmpeg_next::codec::Parameters::new();
     unsafe {
         let aps = audio_params.as_mut_ptr();
         (*aps).codec_id = out_audio_codec.id().into();
+        (*aps).codec_type = ffmpeg_next::ffi::AVMediaType::AVMEDIA_TYPE_AUDIO;
+        (*aps).frame_size = (emu.get_audio_sample_rate() / emu.get_video_fps().floor()) as i32;
         (*aps).sample_rate = emu.get_audio_sample_rate() as i32;
         (*aps).channels = 2;
     };
     out_audio_ctx.set_parameters(audio_params).unwrap();
-    out_audio_ctx.set_time_base(ffmpeg_next::util::rational::Rational::new(
+    // out_audio_ctx.set_time_base(ffmpeg_next::util::rational::Rational::new(
-        1,
+    //     1,
-        emu.get_audio_sample_rate() as i32,
+    //     emu.get_audio_sample_rate() as i32,
-    ));
+    // ));
     let mut out_audio = output.add_stream_with(&out_audio_ctx).unwrap();
-    out_audio.set_time_base(ffmpeg_next::util::rational::Rational::new(
+    let mut encoded_audio = ffmpeg_next::Packet::empty();
-        1,
+    // out_audio.set_time_base(ffmpeg_next::util::rational::Rational::new(
-        emu.get_audio_sample_rate() as i32,
+    //     1,
-    ));
+    //     emu.get_audio_sample_rate() as i32,
-    let audio_time_base = out_audio.time_base();
+    // ));
-    dbg!(audio_time_base);
+    let audio_time_base =
+        ffmpeg_next::util::rational::Rational::new(1, emu.get_audio_sample_rate() as i32);
+    // dbg!(audio_time_base);
     let mut out_audio_enc = out_audio_ctx.encoder().audio().unwrap();
     out_audio_enc.set_channels(2);
     out_audio_enc.set_format(ffmpeg_next::format::Sample::F32(
@@ -127,86 +113,130 @@ fn main() {
         out_video_enc.width(),
         out_video_enc.height(),
     );
+    let mut out_rgbframe = FFVFrame::new(
+        ffmpeg_next::format::Pixel::RGB24,
+        u32::try_from(w).unwrap(),
+        u32::try_from(h).unwrap(),
+    );
     let mut out_aframe = FFAFrame::new(
         out_audio_enc.format(),
         out_audio_enc.frame_size() as usize,
         out_audio_enc.channel_layout(),
     );
-    dbg!(
+    let mut out_i16frame = FFAFrame::new(
-        out_audio_enc.channels(),
+        ffmpeg_next::format::Sample::I16(ffmpeg_next::format::sample::Type::Packed),
+        out_audio_enc.frame_size() as usize,
         out_audio_enc.channel_layout(),
-        out_audio_enc.format()
     );
-    dbg!(out_aframe.samples());
+    // dbg!(
-    dbg!(out_aframe.data(0).len(), out_aframe.data(1).len());
+    //     out_audio_enc.channels(),
+    //     out_audio_enc.channel_layout(),
+    //     out_audio_enc.format()
+    // );
+    // dbg!(out_aframe.samples());
+    // dbg!(out_aframe.data(0).len(), out_aframe.data(1).len());
     output.write_header().unwrap();
     assert!(emu.load(&rply.initial_state));
+    let video_stream_time_base = output.stream(0).unwrap().time_base();
+    let audio_stream_time_base = output.stream(1).unwrap().time_base();
+    encoded_video.set_time_base(video_stream_time_base);
+    encoded_audio.set_time_base(audio_stream_time_base);
+
     let mut frame = Frame::default();
     let mut fb = vec![0_u8; w * h * 3];
-
+    let mut converter = img_conv(
+        (u32::try_from(w).unwrap(), u32::try_from(h).unwrap()),
+        out_rgbframe.format(),
+        out_video_enc.format(),
+    )
+    .unwrap();
+    let mut audio_buf = ringbuf::LocalRb::new(out_aframe.samples() * 2 * 4);
+    let mut frame_audio_buf = vec![0_i16; out_aframe.samples() * 2];
+    let mut audio_frame = 0;
     while let Ok(()) = rply
         .read_frame(&mut frame)
         .inspect_err(|e| println!("Err: {e}"))
     {
         use ffmpeg_next::util::mathematics::Rescale;
-        //println!("FRAME");
         let buttons = frame_to_buttons(&frame);
         emu.run(buttons);
         emu.copy_framebuffer_rgb888(&mut fb).unwrap();
+        if !frame.checkpoint_bytes.is_empty() {
+            // println!("Load CP at {frame_num}");
+            assert!(emu.load(&frame.checkpoint_bytes));
+        }
         // output one frame of video/audio, set_pts
         // copy video to out_vframe
-        copy_video(&fb, &mut out_vframe);
+        copy_video(&fb, &mut converter, &mut out_rgbframe, &mut out_vframe);
         let frame_num = i64::try_from(rply.frame_number).unwrap();
-        out_vframe.set_pts(Some(
+        let frame_pts = frame_num.rescale(emu_time_base, out_video_enc.time_base());
-            frame_num.rescale(emu_time_base, out_video_enc.time_base()),
+        // let mut rgb = image::RgbImage::new(w as u32, h as u32);
-        ));
+        // rgb.clone_from_slice(&fb);
+        // rgb.save(format!("test/{frame_num}.png")).unwrap();
+        out_vframe.set_pts(Some(frame_pts));
         out_video_enc.send_frame(&out_vframe).unwrap();
         // copy audio to out_aframe, set_pts
         // maybe in a loop?
         #[allow(unused_must_use)]
         emu.peek_audio_sample(|samples| {
-            copy_audio(samples, &mut out_aframe);
+            audio_buf.push_slice_overwrite(samples);
-        });
+            while audio_buf.occupied_len() >= out_aframe.samples() {
-        out_aframe.set_pts(Some(
+                enc_audio = true;
-            frame_num.rescale(emu_time_base, out_audio_enc.time_base()),
+                audio_buf.pop_slice(&mut frame_audio_buf);
-        ));
+                copy_audio(&frame_audio_buf, &mut out_aframe);
-        println!(
+                out_aframe.set_pts(Some(audio_frame));
-            "vtime {} atime {}",
+                audio_frame += out_aframe.samples();
-            out_vframe.pts().unwrap() as f64 / 60.0,
-            out_aframe.pts().unwrap() as f64 / 48000.0
-        );
                 out_audio_enc.send_frame(&out_aframe).unwrap();
-        let mut encoded = ffmpeg_next::Packet::empty();
-        while out_video_enc.receive_packet(&mut encoded).is_ok() {
-            encoded.set_stream(0);
-            encoded.rescale_ts(out_video_enc.time_base(), video_time_base);
-            encoded.write_interleaved(&mut output).unwrap();
             }
-        while out_audio_enc.receive_packet(&mut encoded).is_ok() {
+        });
-            encoded.set_stream(1);
+        // println!(
-            encoded.rescale_ts(out_audio_enc.time_base(), audio_time_base);
+        //     "vtime {} atime {}",
-            encoded.write_interleaved(&mut output).unwrap();
+        //     out_vframe.pts().unwrap() as f64 / 60.0,
+        //     out_aframe.pts().unwrap() as f64 / 48000.0
+        // );
+        while out_video_enc.receive_packet(&mut encoded_video).is_ok() {
+            encoded_video.set_stream(0);
+            encoded_video.rescale_ts(out_video_enc.time_base(), video_stream_time_base);
+            encoded_video.write_interleaved(&mut output).unwrap();
         }
-        if !frame.checkpoint_bytes.is_empty() {
+        while out_audio_enc.receive_packet(&mut encoded_audio).is_ok() {
-            assert!(emu.load(&frame.checkpoint_bytes));
+            encoded_audio.set_stream(1);
+            encoded_audio.rescale_ts(out_audio_enc.time_base(), audio_stream_time_base);
+            encoded_audio.write_interleaved(&mut output).unwrap();
         }
         if Some(rply.frame_number) == rply.header.frame_count() {
             break;
         }
     }
+    while audio_buf.occupied_len() >= out_aframe.samples() {
+        let len = audio_buf.pop_slice(&mut frame_audio_buf);
+        frame_audio_buf[len..].fill(0);
+        out_aframe.set_pts(Some(audio_frame));
+        out_aframe.set_samples(len / 2);
+        audio_frame += out_aframe.samples();
+        copy_audio(&frame_audio_buf, &mut out_aframe);
+        out_audio_enc.send_frame(&out_aframe).unwrap();
+    }
+    while out_audio_enc.receive_packet(&mut encoded_audio).is_ok() {
+        encoded_audio.set_stream(1);
+        encoded_audio.rescale_ts(out_audio_enc.time_base(), audio_stream_time_base);
+        encoded_audio.write_interleaved(&mut output).unwrap();
+    }
+
     out_video_enc.send_eof().unwrap();
     out_audio_enc.send_eof().unwrap();
     output.write_trailer().unwrap();
+    dbg!(output.stream(0).unwrap().time_base());
 }
 
 fn frame_to_buttons(frame: &Frame) -> [retro_rs::Buttons; 2] {
     use retro_rs::Buttons;
-    let mut buttons = [Buttons::default(); 2];
+    let mut buttons = [0_i16; 2];
     for inp in &frame.input_events {
         let port = usize::from(inp.port);
         if port < buttons.len() && inp.device == 1 {
-            buttons[port] = Buttons::from(inp.val);
+            buttons[port] |= inp.val;
         }
     }
-    buttons
+    [Buttons::from(buttons[0]), Buttons::from(buttons[1])]
 }