Refactor, speed up debug builds by avoiding copy_framebuffer step

Cargo.lock

@@ -210,7 +210,7 @@ name = "genvideo"
 version = "0.1.0"
 dependencies = [
  "ffmpeg-next",
- "retro-rs 0.5.4",
+ "retro-rs",
  "ringbuf",
  "rply-codec",
 ]
@@ -474,18 +474,9 @@ checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58"
 
 [[package]]
 name = "retro-rs"
-version = "0.5.3"
+version = "0.5.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb116c907a72a722faf31bc45293bd4748a85109807ae5334001c7c74e188be2"
+checksum = "b2dc1c5993446d4122c2f8ecf21b4e96db47678d139868848dee603511df9e1c"
-dependencies = [
- "libc",
- "libloading",
- "rust-libretro-sys",
-]
-
-[[package]]
-name = "retro-rs"
-version = "0.5.4"
 dependencies = [
  "libc",
  "libloading",
@@ -646,7 +637,7 @@ checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5"
 name = "upgradev0"
 version = "0.1.0"
 dependencies = [
- "retro-rs 0.5.3",
+ "retro-rs",
  "rply-codec",
 ]
 

genvideo/Cargo.toml

@@ -5,6 +5,6 @@ edition = "2024"
 
 [dependencies]
 rply-codec = { path = "../codec" }
-retro-rs = { version = "0.5.4", default-features=false, path = "../../retro-rs" }
+retro-rs = { version = "0.5.5", default-features=false }
 ffmpeg-next = "8.0.0"
 ringbuf = "0.4.8"

genvideo/src/main.rs

@@ -1,4 +1,6 @@
+use ffmpeg_next::util::{mathematics::Rescale, rational::Rational};
 use ffmpeg_next::{
+    format::context::Output as FFOut,
     software::converter as img_conv,
     util::frame::{Audio as FFAFrame, Video as FFVFrame},
 };
@@ -46,19 +48,227 @@ fn copy_audio(samples: &[i16], frame: &mut FFAFrame) {
         frame.plane_mut(1)[i] = f32::from(*r) / BOUND;
     }
 }
-fn copy_video(
+struct VideoState {
-    fb: &[u8],
+    out_video_enc: ffmpeg_next::encoder::video::Encoder,
-    converter: &mut ffmpeg_next::software::scaling::Context,
+    out_vframe: FFVFrame,
-    rgbframe: &mut FFVFrame,
+    out_rgbframe: FFVFrame,
-    outframe: &mut FFVFrame,
+    encoded_video: ffmpeg_next::Packet,
-) {
+    converter: ffmpeg_next::software::scaling::Context,
-    rgbframe.data_mut(0).copy_from_slice(fb);
+    emu_time_base: Rational,
-    converter.run(rgbframe, outframe).unwrap();
+    native_pixel_format: bool,
+}
+
+impl VideoState {
+    fn new(
+        emu_time_base: Rational,
+        w: usize,
+        h: usize,
+        pixel_format: retro_rs::libretro::retro_pixel_format,
+        output: &mut FFOut,
+    ) -> Self {
+        let out_video_codec = ffmpeg_next::encoder::find(ffmpeg_next::codec::Id::H264).unwrap();
+        let mut out_video_ctx =
+            ffmpeg_next::codec::context::Context::new_with_codec(out_video_codec);
+        // out_video_ctx.set_time_base(emu_time_base);
+        let mut video_params = ffmpeg_next::codec::Parameters::new();
+        unsafe {
+            let vps = video_params.as_mut_ptr();
+            (*vps).width = i32::try_from(w).unwrap();
+            (*vps).height = i32::try_from(h).unwrap();
+            (*vps).codec_id = out_video_codec.id().into();
+            (*vps).codec_type = ffmpeg_next::ffi::AVMediaType::AVMEDIA_TYPE_VIDEO;
+        };
+        out_video_ctx.set_parameters(video_params).unwrap();
+        let _out_video = output.add_stream_with(&out_video_ctx).unwrap();
+        let encoded_video = ffmpeg_next::Packet::empty();
+        // out_video.set_time_base(emu_time_base);
+        let mut out_video_enc = out_video_ctx.encoder().video().unwrap();
+        out_video_enc.set_format(ffmpeg_next::format::Pixel::YUV420P);
+        out_video_enc.set_width(u32::try_from(w).unwrap());
+        out_video_enc.set_height(u32::try_from(h).unwrap());
+        out_video_enc.set_time_base(emu_time_base);
+        let out_video_enc = out_video_enc.open().unwrap();
+        let out_vframe = FFVFrame::new(
+            out_video_enc.format(),
+            out_video_enc.width(),
+            out_video_enc.height(),
+        );
+        /* TODO: Consider using the pixel format of the emulator  here and avoid copying as rgb888 */
+        let (copy_format, is_native) = match pixel_format {
+            retro_rs::libretro::retro_pixel_format::RETRO_PIXEL_FORMAT_0RGB1555 => {
+                (ffmpeg_next::format::Pixel::RGB555, true)
+            }
+            retro_rs::libretro::retro_pixel_format::RETRO_PIXEL_FORMAT_XRGB8888 => {
+                (ffmpeg_next::format::Pixel::ZRGB, true)
+            }
+            retro_rs::libretro::retro_pixel_format::RETRO_PIXEL_FORMAT_RGB565 => {
+                (ffmpeg_next::format::Pixel::RGB565, true)
+            }
+            _other => (ffmpeg_next::format::Pixel::RGB24, false),
+        };
+        let out_rgbframe = FFVFrame::new(
+            copy_format,
+            u32::try_from(w).unwrap(),
+            u32::try_from(h).unwrap(),
+        );
+
+        let converter = img_conv(
+            (u32::try_from(w).unwrap(), u32::try_from(h).unwrap()),
+            out_rgbframe.format(),
+            out_video_enc.format(),
+        )
+        .unwrap();
+        Self {
+            out_video_enc,
+            out_vframe,
+            out_rgbframe,
+            encoded_video,
+            converter,
+            emu_time_base,
+            native_pixel_format: is_native,
+        }
+    }
+    fn writeout(&mut self, output: &mut FFOut) {
+        let output_time_base = output.stream(0).unwrap().time_base();
+        while self
+            .out_video_enc
+            .receive_packet(&mut self.encoded_video)
+            .is_ok()
+        {
+            self.encoded_video.set_stream(0);
+            self.encoded_video
+                .rescale_ts(self.out_video_enc.time_base(), output_time_base);
+            self.encoded_video.write_interleaved(output).unwrap();
+        }
+    }
+    fn send_frame(&mut self, emu: &Emulator, frame_num: u64, output: &mut FFOut) {
+        // output one frame of video/audio, set_pts
+        // copy video to out_vframe
+        if self.native_pixel_format {
+            emu.peek_framebuffer(|fb| {
+                self.out_rgbframe.data_mut(0).copy_from_slice(fb);
+            })
+            .unwrap();
+        } else {
+            emu.copy_framebuffer_rgb888(self.out_rgbframe.data_mut(0))
+                .unwrap();
+        }
+        self.converter
+            .run(&self.out_rgbframe, &mut self.out_vframe)
+            .unwrap();
+        let frame_num = i64::try_from(frame_num).unwrap();
+        let frame_pts = frame_num.rescale(self.emu_time_base, self.out_video_enc.time_base());
+        self.out_vframe.set_pts(Some(frame_pts));
+        self.out_video_enc.send_frame(&self.out_vframe).unwrap();
+        self.writeout(output);
+    }
+    fn drain(&mut self, output: &mut FFOut) {
+        self.out_video_enc.send_eof().unwrap();
+        self.writeout(output);
+    }
+}
+
+struct AudioState {
+    out_audio_enc: ffmpeg_next::encoder::audio::Encoder,
+    out_aframe: FFAFrame,
+    encoded_audio: ffmpeg_next::Packet,
+    audio_buf: ringbuf::LocalRb<ringbuf::storage::Heap<i16>>,
+    frame_audio_buf: Vec<i16>,
+    audio_frame: i64,
+}
+
+impl AudioState {
+    fn new(audio_sample_rate: i32, output: &mut FFOut) -> Self {
+        let out_audio_codec = ffmpeg_next::encoder::find(ffmpeg_next::codec::Id::AAC).unwrap();
+        let mut out_audio_ctx =
+            ffmpeg_next::codec::context::Context::new_with_codec(out_audio_codec);
+        // out_audio_ctx.debug(Debug::all());
+        let mut audio_params = ffmpeg_next::codec::Parameters::new();
+        unsafe {
+            let aps = audio_params.as_mut_ptr();
+            (*aps).codec_id = out_audio_codec.id().into();
+            (*aps).codec_type = ffmpeg_next::ffi::AVMediaType::AVMEDIA_TYPE_AUDIO;
+            (*aps).sample_rate = audio_sample_rate;
+            (*aps).frame_size = 1024 * 2 * 4;
+            (*aps).channels = 2;
+        };
+        out_audio_ctx.set_parameters(audio_params).unwrap();
+        let _out_audio = output.add_stream_with(&out_audio_ctx).unwrap();
+        let encoded_audio = ffmpeg_next::Packet::empty();
+        let audio_time_base = Rational::new(1, audio_sample_rate);
+        let mut out_audio_enc = out_audio_ctx.encoder().audio().unwrap();
+        out_audio_enc.set_channels(2);
+        out_audio_enc.set_format(ffmpeg_next::format::Sample::F32(
+            ffmpeg_next::format::sample::Type::Planar,
+        ));
+        out_audio_enc.set_channel_layout(ffmpeg_next::ChannelLayout::STEREO);
+        out_audio_enc.set_time_base(audio_time_base);
+        out_audio_enc.set_rate(audio_sample_rate);
+        let out_audio_enc = out_audio_enc.open().unwrap();
+        let out_aframe = FFAFrame::new(
+            out_audio_enc.format(),
+            out_audio_enc.frame_size() as usize,
+            out_audio_enc.channel_layout(),
+        );
+        let audio_buf = ringbuf::LocalRb::new(out_aframe.samples() * 2 * 20);
+        let frame_audio_buf = vec![0_i16; out_aframe.samples() * 2];
+
+        Self {
+            out_audio_enc,
+            out_aframe,
+            encoded_audio,
+            audio_buf,
+            frame_audio_buf,
+            audio_frame: 0,
+        }
+    }
+    fn writeout(&mut self, output: &mut FFOut) {
+        let output_time_base = output.stream(1).unwrap().time_base();
+        while self
+            .out_audio_enc
+            .receive_packet(&mut self.encoded_audio)
+            .is_ok()
+        {
+            self.encoded_audio.set_stream(1);
+            self.encoded_audio
+                .rescale_ts(self.out_audio_enc.time_base(), output_time_base);
+            self.encoded_audio.write_interleaved(output).unwrap();
+        }
+    }
+    fn send_frames(&mut self, emu: &Emulator, output: &mut FFOut) {
+        #[allow(unused_must_use)]
+        emu.peek_audio_sample(|samples| {
+            self.audio_buf.push_slice_overwrite(samples);
+            while self.audio_buf.occupied_len() >= self.out_aframe.samples() * 2 {
+                assert_eq!(
+                    self.audio_buf.pop_slice(&mut self.frame_audio_buf),
+                    self.frame_audio_buf.len()
+                );
+                copy_audio(&self.frame_audio_buf, &mut self.out_aframe);
+                self.out_aframe.set_pts(Some(self.audio_frame));
+                self.audio_frame += i64::try_from(self.out_aframe.samples()).unwrap();
+                self.out_audio_enc.send_frame(&self.out_aframe).unwrap();
+            }
+        });
+        self.writeout(output);
+    }
+    fn drain(&mut self, output: &mut FFOut) {
+        while self.audio_buf.occupied_len() >= self.out_aframe.samples() {
+            let len = self.audio_buf.pop_slice(&mut self.frame_audio_buf);
+            self.frame_audio_buf[len..].fill(0);
+            self.out_aframe.set_pts(Some(self.audio_frame));
+            self.audio_frame += i64::try_from(len / 2).unwrap();
+            copy_audio(&self.frame_audio_buf, &mut self.out_aframe);
+            self.out_audio_enc.send_frame(&self.out_aframe).unwrap();
+        }
+        self.out_audio_enc.send_eof().unwrap();
+        self.writeout(output);
+    }
 }
 
 fn main() {
     ffmpeg_next::init().unwrap();
-    ffmpeg_next::log::set_level(ffmpeg_next::log::Level::Trace);
+    ffmpeg_next::log::set_level(ffmpeg_next::log::Level::Warning);
     let args: Vec<_> = std::env::args().collect();
     let file =
         std::fs::File::open(args.get(1).unwrap_or(&"examples/bobl.replay".to_string())).unwrap();
@@ -79,174 +289,44 @@ fn main() {
     }
     // run emu a tick to make sure we have right frame sizes, etc
     emu.run([retro_rs::Buttons::default(); 2]);
-
     let (w, h) = emu.framebuffer_size();
+    let pixel_format = emu.pixel_format();
+    assert!(emu.load(&rply.initial_state));
 
     let mut output = ffmpeg_next::format::output(&outfile).unwrap();
-    let emu_time_base =
+    let emu_time_base = Rational::new(1, emu.get_video_fps().to_i32().unwrap());
-        ffmpeg_next::util::rational::Rational::new(1, emu.get_video_fps().to_i32().unwrap());
+    let audio_sample_rate = emu.get_audio_sample_rate().to_i32().unwrap();
-    let out_video_codec = ffmpeg_next::encoder::find(ffmpeg_next::codec::Id::H264).unwrap();
+    let mut video_state = VideoState::new(emu_time_base, w, h, pixel_format, &mut output);
-    let mut out_video_ctx = ffmpeg_next::codec::context::Context::new_with_codec(out_video_codec);
+    let mut audio_state = AudioState::new(audio_sample_rate, &mut output);
-    // out_video_ctx.set_time_base(emu_time_base);
-    let mut video_params = ffmpeg_next::codec::Parameters::new();
-    unsafe {
-        let vps = video_params.as_mut_ptr();
-        (*vps).width = i32::try_from(w).unwrap();
-        (*vps).height = i32::try_from(h).unwrap();
-        (*vps).codec_id = out_video_codec.id().into();
-        (*vps).codec_type = ffmpeg_next::ffi::AVMediaType::AVMEDIA_TYPE_VIDEO;
-    };
-    out_video_ctx.set_parameters(video_params).unwrap();
-    let _out_video = output.add_stream_with(&out_video_ctx).unwrap();
-    let mut encoded_video = ffmpeg_next::Packet::empty();
-    // out_video.set_time_base(emu_time_base);
-    let mut out_video_enc = out_video_ctx.encoder().video().unwrap();
-    out_video_enc.set_format(ffmpeg_next::format::Pixel::YUV420P);
-    out_video_enc.set_width(u32::try_from(w).unwrap());
-    out_video_enc.set_height(u32::try_from(h).unwrap());
-    out_video_enc.set_time_base(emu_time_base);
-    let mut out_video_enc = out_video_enc.open().unwrap();
-    let out_audio_codec = ffmpeg_next::encoder::find(ffmpeg_next::codec::Id::AAC).unwrap();
-    let mut out_audio_ctx = ffmpeg_next::codec::context::Context::new_with_codec(out_audio_codec);
-    // out_audio_ctx.debug(Debug::all());
-    let mut audio_params = ffmpeg_next::codec::Parameters::new();
-    unsafe {
-        let aps = audio_params.as_mut_ptr();
-        (*aps).codec_id = out_audio_codec.id().into();
-        (*aps).codec_type = ffmpeg_next::ffi::AVMediaType::AVMEDIA_TYPE_AUDIO;
-        (*aps).frame_size = (emu.get_audio_sample_rate() / emu.get_video_fps().floor())
-            .to_i32()
-            .unwrap();
-        (*aps).sample_rate = emu.get_audio_sample_rate().to_i32().unwrap();
-        (*aps).channels = 2;
-    };
-    out_audio_ctx.set_parameters(audio_params).unwrap();
-    // out_audio_ctx.set_time_base(ffmpeg_next::util::rational::Rational::new(
-    //     1,
-    //     emu.get_audio_sample_rate() as i32,
-    // ));
-    let _out_audio = output.add_stream_with(&out_audio_ctx).unwrap();
-    let mut encoded_audio = ffmpeg_next::Packet::empty();
-    // out_audio.set_time_base(ffmpeg_next::util::rational::Rational::new(
-    //     1,
-    //     emu.get_audio_sample_rate() as i32,
-    // ));
-    let audio_time_base = ffmpeg_next::util::rational::Rational::new(
-        1,
-        emu.get_audio_sample_rate().to_i32().unwrap(),
-    );
-    let mut out_audio_enc = out_audio_ctx.encoder().audio().unwrap();
-    out_audio_enc.set_channels(2);
-    out_audio_enc.set_format(ffmpeg_next::format::Sample::F32(
-        ffmpeg_next::format::sample::Type::Planar,
-    ));
-    out_audio_enc.set_channel_layout(ffmpeg_next::ChannelLayout::STEREO);
-    out_audio_enc.set_time_base(audio_time_base);
-    out_audio_enc.set_rate(emu.get_audio_sample_rate().to_i32().unwrap());
-    let mut out_audio_enc = out_audio_enc.open().unwrap();
-    let mut out_vframe = FFVFrame::new(
-        out_video_enc.format(),
-        out_video_enc.width(),
-        out_video_enc.height(),
-    );
-    let mut out_rgbframe = FFVFrame::new(
-        ffmpeg_next::format::Pixel::RGB24,
-        u32::try_from(w).unwrap(),
-        u32::try_from(h).unwrap(),
-    );
-    let mut out_aframe = FFAFrame::new(
-        out_audio_enc.format(),
-        out_audio_enc.frame_size() as usize,
-        out_audio_enc.channel_layout(),
-    );
     output.write_header().unwrap();
-    assert!(emu.load(&rply.initial_state));
     let video_stream_time_base = output.stream(0).unwrap().time_base();
     let audio_stream_time_base = output.stream(1).unwrap().time_base();
-    encoded_video.set_time_base(video_stream_time_base);
+    video_state
-    encoded_audio.set_time_base(audio_stream_time_base);
+        .encoded_video
+        .set_time_base(video_stream_time_base);
+    audio_state
+        .encoded_audio
+        .set_time_base(audio_stream_time_base);
 
     let mut frame = Frame::default();
-    let mut fb = vec![0_u8; w * h * 3];
-    let mut converter = img_conv(
-        (u32::try_from(w).unwrap(), u32::try_from(h).unwrap()),
-        out_rgbframe.format(),
-        out_video_enc.format(),
-    )
-    .unwrap();
-    let mut audio_buf = ringbuf::LocalRb::new(out_aframe.samples() * 2 * 20);
-    let mut frame_audio_buf = vec![0_i16; out_aframe.samples() * 2];
-    let mut audio_frame = 0;
     while let Ok(()) = rply
         .read_frame(&mut frame)
         .inspect_err(|e| println!("Err: {e}"))
     {
-        use ffmpeg_next::util::mathematics::Rescale;
         let buttons = frame_to_buttons(&frame);
         emu.run(buttons);
-        emu.copy_framebuffer_rgb888(&mut fb).unwrap();
+        video_state.send_frame(&emu, rply.frame_number, &mut output);
+        audio_state.send_frames(&emu, &mut output);
         if !frame.checkpoint_bytes.is_empty() {
             assert!(emu.load(&frame.checkpoint_bytes));
         }
-        // output one frame of video/audio, set_pts
+
-        // copy video to out_vframe
-        copy_video(&fb, &mut converter, &mut out_rgbframe, &mut out_vframe);
-        let frame_num = i64::try_from(rply.frame_number).unwrap();
-        let frame_pts = frame_num.rescale(emu_time_base, out_video_enc.time_base());
-        out_vframe.set_pts(Some(frame_pts));
-        out_video_enc.send_frame(&out_vframe).unwrap();
-        // copy audio to out_aframe, set_pts
-        // maybe in a loop?
-        #[allow(unused_must_use)]
-        emu.peek_audio_sample(|samples| {
-            audio_buf.push_slice_overwrite(samples);
-            while audio_buf.occupied_len() >= out_aframe.samples() * 2 {
-                assert_eq!(
-                    audio_buf.pop_slice(&mut frame_audio_buf),
-                    frame_audio_buf.len()
-                );
-                copy_audio(&frame_audio_buf, &mut out_aframe);
-                out_aframe.set_pts(Some(audio_frame));
-                audio_frame += i64::try_from(out_aframe.samples()).unwrap();
-                out_audio_enc.send_frame(&out_aframe).unwrap();
-            }
-        });
-        while out_video_enc.receive_packet(&mut encoded_video).is_ok() {
-            encoded_video.set_stream(0);
-            encoded_video.rescale_ts(out_video_enc.time_base(), video_stream_time_base);
-            encoded_video.write_interleaved(&mut output).unwrap();
-        }
-        while out_audio_enc.receive_packet(&mut encoded_audio).is_ok() {
-            encoded_audio.set_stream(1);
-            encoded_audio.rescale_ts(out_audio_enc.time_base(), audio_stream_time_base);
-            encoded_audio.write_interleaved(&mut output).unwrap();
-        }
         if Some(rply.frame_number) == rply.header.frame_count() {
             break;
         }
     }
-    while audio_buf.occupied_len() >= out_aframe.samples() {
+    audio_state.drain(&mut output);
-        let len = audio_buf.pop_slice(&mut frame_audio_buf);
+    video_state.drain(&mut output);
-        frame_audio_buf[len..].fill(0);
-        out_aframe.set_pts(Some(audio_frame));
-        audio_frame += i64::try_from(len / 2).unwrap();
-        copy_audio(&frame_audio_buf, &mut out_aframe);
-        out_audio_enc.send_frame(&out_aframe).unwrap();
-    }
-
-    out_video_enc.send_eof().unwrap();
-    out_audio_enc.send_eof().unwrap();
-
-    while out_video_enc.receive_packet(&mut encoded_video).is_ok() {
-        encoded_video.set_stream(0);
-        encoded_video.rescale_ts(out_video_enc.time_base(), video_stream_time_base);
-        encoded_video.write_interleaved(&mut output).unwrap();
-    }
-    while out_audio_enc.receive_packet(&mut encoded_audio).is_ok() {
-        encoded_audio.set_stream(1);
-        encoded_audio.rescale_ts(out_audio_enc.time_base(), audio_stream_time_base);
-        encoded_audio.write_interleaved(&mut output).unwrap();
-    }
     output.write_trailer().unwrap();
 }