shftgroup/rply-codec
3
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Refactor, speed up debug builds by avoiding copy_framebuffer step

Browse Source
This commit is contained in:
Joseph C. Osborn
2025-11-06 10:52:31 -08:00
parent f2e80a04b6
commit 48bd011b9e
3 changed files with 241 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+4 -13
View File
@@ -210,7 +210,7 @@ name = "genvideo"
version = "0.1.0"
dependencies = [
"ffmpeg-next",
"retro-rs 0.5.4",
"retro-rs",
"ringbuf",
"rply-codec",
]
@@ -474,18 +474,9 @@ checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58"
[[package]]
name = "retro-rs"
version = "0.5.3"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb116c907a72a722faf31bc45293bd4748a85109807ae5334001c7c74e188be2"
dependencies = [
"libc",
"libloading",
"rust-libretro-sys",
]
[[package]]
name = "retro-rs"
version = "0.5.4"
checksum = "b2dc1c5993446d4122c2f8ecf21b4e96db47678d139868848dee603511df9e1c"
dependencies = [
"libc",
"libloading",
@@ -646,7 +637,7 @@ checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5"
name = "upgradev0"
version = "0.1.0"
dependencies = [
"retro-rs 0.5.3",
"retro-rs",
"rply-codec",
]
genvideo/Cargo.toml
+1 -1
View File
@@ -5,6 +5,6 @@ edition = "2024"
[dependencies]
rply-codec = { path = "../codec" }
retro-rs = { version = "0.5.4", default-features=false, path = "../../retro-rs" }
retro-rs = { version = "0.5.5", default-features=false }
ffmpeg-next = "8.0.0"
ringbuf = "0.4.8"
genvideo/src/main.rs
+236 -156
View File
@@ -1,4 +1,6 @@
use ffmpeg_next::util::{mathematics::Rescale, rational::Rational};
use ffmpeg_next::{
format::context::Output as FFOut,
software::converter as img_conv,
util::frame::{Audio as FFAFrame, Video as FFVFrame},
};
@@ -46,19 +48,227 @@ fn copy_audio(samples: &[i16], frame: &mut FFAFrame) {
frame.plane_mut(1)[i] = f32::from(*r) / BOUND;
}
}
fn copy_video(
fb: &[u8],
converter: &mut ffmpeg_next::software::scaling::Context,
rgbframe: &mut FFVFrame,
outframe: &mut FFVFrame,
) {
rgbframe.data_mut(0).copy_from_slice(fb);
converter.run(rgbframe, outframe).unwrap();
struct VideoState {
out_video_enc: ffmpeg_next::encoder::video::Encoder,
out_vframe: FFVFrame,
out_rgbframe: FFVFrame,
encoded_video: ffmpeg_next::Packet,
converter: ffmpeg_next::software::scaling::Context,
emu_time_base: Rational,
native_pixel_format: bool,
}
impl VideoState {
fn new(
emu_time_base: Rational,
w: usize,
h: usize,
pixel_format: retro_rs::libretro::retro_pixel_format,
output: &mut FFOut,
) -> Self {
let out_video_codec = ffmpeg_next::encoder::find(ffmpeg_next::codec::Id::H264).unwrap();
let mut out_video_ctx =
ffmpeg_next::codec::context::Context::new_with_codec(out_video_codec);
// out_video_ctx.set_time_base(emu_time_base);
let mut video_params = ffmpeg_next::codec::Parameters::new();
unsafe {
let vps = video_params.as_mut_ptr();
(*vps).width = i32::try_from(w).unwrap();
(*vps).height = i32::try_from(h).unwrap();
(*vps).codec_id = out_video_codec.id().into();
(*vps).codec_type = ffmpeg_next::ffi::AVMediaType::AVMEDIA_TYPE_VIDEO;
};
out_video_ctx.set_parameters(video_params).unwrap();
let _out_video = output.add_stream_with(&out_video_ctx).unwrap();
let encoded_video = ffmpeg_next::Packet::empty();
// out_video.set_time_base(emu_time_base);
let mut out_video_enc = out_video_ctx.encoder().video().unwrap();
out_video_enc.set_format(ffmpeg_next::format::Pixel::YUV420P);
out_video_enc.set_width(u32::try_from(w).unwrap());
out_video_enc.set_height(u32::try_from(h).unwrap());
out_video_enc.set_time_base(emu_time_base);
let out_video_enc = out_video_enc.open().unwrap();
let out_vframe = FFVFrame::new(
out_video_enc.format(),
out_video_enc.width(),
out_video_enc.height(),
);
/* TODO: Consider using the pixel format of the emulator here and avoid copying as rgb888 */
let (copy_format, is_native) = match pixel_format {
retro_rs::libretro::retro_pixel_format::RETRO_PIXEL_FORMAT_0RGB1555 => {
(ffmpeg_next::format::Pixel::RGB555, true)
}
retro_rs::libretro::retro_pixel_format::RETRO_PIXEL_FORMAT_XRGB8888 => {
(ffmpeg_next::format::Pixel::ZRGB, true)
}
retro_rs::libretro::retro_pixel_format::RETRO_PIXEL_FORMAT_RGB565 => {
(ffmpeg_next::format::Pixel::RGB565, true)
}
_other => (ffmpeg_next::format::Pixel::RGB24, false),
};
let out_rgbframe = FFVFrame::new(
copy_format,
u32::try_from(w).unwrap(),
u32::try_from(h).unwrap(),
);
let converter = img_conv(
(u32::try_from(w).unwrap(), u32::try_from(h).unwrap()),
out_rgbframe.format(),
out_video_enc.format(),
)
.unwrap();
Self {
out_video_enc,
out_vframe,
out_rgbframe,
encoded_video,
converter,
emu_time_base,
native_pixel_format: is_native,
}
}
fn writeout(&mut self, output: &mut FFOut) {
let output_time_base = output.stream(0).unwrap().time_base();
while self
.out_video_enc
.receive_packet(&mut self.encoded_video)
.is_ok()
{
self.encoded_video.set_stream(0);
self.encoded_video
.rescale_ts(self.out_video_enc.time_base(), output_time_base);
self.encoded_video.write_interleaved(output).unwrap();
}
}
fn send_frame(&mut self, emu: &Emulator, frame_num: u64, output: &mut FFOut) {
// output one frame of video/audio, set_pts
// copy video to out_vframe
if self.native_pixel_format {
emu.peek_framebuffer(|fb| {
self.out_rgbframe.data_mut(0).copy_from_slice(fb);
})
.unwrap();
} else {
emu.copy_framebuffer_rgb888(self.out_rgbframe.data_mut(0))
.unwrap();
}
self.converter
.run(&self.out_rgbframe, &mut self.out_vframe)
.unwrap();
let frame_num = i64::try_from(frame_num).unwrap();
let frame_pts = frame_num.rescale(self.emu_time_base, self.out_video_enc.time_base());
self.out_vframe.set_pts(Some(frame_pts));
self.out_video_enc.send_frame(&self.out_vframe).unwrap();
self.writeout(output);
}
fn drain(&mut self, output: &mut FFOut) {
self.out_video_enc.send_eof().unwrap();
self.writeout(output);
}
}
struct AudioState {
out_audio_enc: ffmpeg_next::encoder::audio::Encoder,
out_aframe: FFAFrame,
encoded_audio: ffmpeg_next::Packet,
audio_buf: ringbuf::LocalRb<ringbuf::storage::Heap<i16>>,
frame_audio_buf: Vec<i16>,
audio_frame: i64,
}
impl AudioState {
fn new(audio_sample_rate: i32, output: &mut FFOut) -> Self {
let out_audio_codec = ffmpeg_next::encoder::find(ffmpeg_next::codec::Id::AAC).unwrap();
let mut out_audio_ctx =
ffmpeg_next::codec::context::Context::new_with_codec(out_audio_codec);
// out_audio_ctx.debug(Debug::all());
let mut audio_params = ffmpeg_next::codec::Parameters::new();
unsafe {
let aps = audio_params.as_mut_ptr();
(*aps).codec_id = out_audio_codec.id().into();
(*aps).codec_type = ffmpeg_next::ffi::AVMediaType::AVMEDIA_TYPE_AUDIO;
(*aps).sample_rate = audio_sample_rate;
(*aps).frame_size = 1024 * 2 * 4;
(*aps).channels = 2;
};
out_audio_ctx.set_parameters(audio_params).unwrap();
let _out_audio = output.add_stream_with(&out_audio_ctx).unwrap();
let encoded_audio = ffmpeg_next::Packet::empty();
let audio_time_base = Rational::new(1, audio_sample_rate);
let mut out_audio_enc = out_audio_ctx.encoder().audio().unwrap();
out_audio_enc.set_channels(2);
out_audio_enc.set_format(ffmpeg_next::format::Sample::F32(
ffmpeg_next::format::sample::Type::Planar,
));
out_audio_enc.set_channel_layout(ffmpeg_next::ChannelLayout::STEREO);
out_audio_enc.set_time_base(audio_time_base);
out_audio_enc.set_rate(audio_sample_rate);
let out_audio_enc = out_audio_enc.open().unwrap();
let out_aframe = FFAFrame::new(
out_audio_enc.format(),
out_audio_enc.frame_size() as usize,
out_audio_enc.channel_layout(),
);
let audio_buf = ringbuf::LocalRb::new(out_aframe.samples() * 2 * 20);
let frame_audio_buf = vec![0_i16; out_aframe.samples() * 2];
Self {
out_audio_enc,
out_aframe,
encoded_audio,
audio_buf,
frame_audio_buf,
audio_frame: 0,
}
}
fn writeout(&mut self, output: &mut FFOut) {
let output_time_base = output.stream(1).unwrap().time_base();
while self
.out_audio_enc
.receive_packet(&mut self.encoded_audio)
.is_ok()
{
self.encoded_audio.set_stream(1);
self.encoded_audio
.rescale_ts(self.out_audio_enc.time_base(), output_time_base);
self.encoded_audio.write_interleaved(output).unwrap();
}
}
fn send_frames(&mut self, emu: &Emulator, output: &mut FFOut) {
#[allow(unused_must_use)]
emu.peek_audio_sample(|samples| {
self.audio_buf.push_slice_overwrite(samples);
while self.audio_buf.occupied_len() >= self.out_aframe.samples() * 2 {
assert_eq!(
self.audio_buf.pop_slice(&mut self.frame_audio_buf),
self.frame_audio_buf.len()
);
copy_audio(&self.frame_audio_buf, &mut self.out_aframe);
self.out_aframe.set_pts(Some(self.audio_frame));
self.audio_frame += i64::try_from(self.out_aframe.samples()).unwrap();
self.out_audio_enc.send_frame(&self.out_aframe).unwrap();
}
});
self.writeout(output);
}
fn drain(&mut self, output: &mut FFOut) {
while self.audio_buf.occupied_len() >= self.out_aframe.samples() {
let len = self.audio_buf.pop_slice(&mut self.frame_audio_buf);
self.frame_audio_buf[len..].fill(0);
self.out_aframe.set_pts(Some(self.audio_frame));
self.audio_frame += i64::try_from(len / 2).unwrap();
copy_audio(&self.frame_audio_buf, &mut self.out_aframe);
self.out_audio_enc.send_frame(&self.out_aframe).unwrap();
}
self.out_audio_enc.send_eof().unwrap();
self.writeout(output);
}
}
fn main() {
ffmpeg_next::init().unwrap();
ffmpeg_next::log::set_level(ffmpeg_next::log::Level::Trace);
ffmpeg_next::log::set_level(ffmpeg_next::log::Level::Warning);
let args: Vec<_> = std::env::args().collect();
let file =
std::fs::File::open(args.get(1).unwrap_or(&"examples/bobl.replay".to_string())).unwrap();
@@ -79,174 +289,44 @@ fn main() {
}
// run emu a tick to make sure we have right frame sizes, etc
emu.run([retro_rs::Buttons::default(); 2]);
let (w, h) = emu.framebuffer_size();
let pixel_format = emu.pixel_format();
assert!(emu.load(&rply.initial_state));
let mut output = ffmpeg_next::format::output(&outfile).unwrap();
let emu_time_base =
ffmpeg_next::util::rational::Rational::new(1, emu.get_video_fps().to_i32().unwrap());
let out_video_codec = ffmpeg_next::encoder::find(ffmpeg_next::codec::Id::H264).unwrap();
let mut out_video_ctx = ffmpeg_next::codec::context::Context::new_with_codec(out_video_codec);
// out_video_ctx.set_time_base(emu_time_base);
let mut video_params = ffmpeg_next::codec::Parameters::new();
unsafe {
let vps = video_params.as_mut_ptr();
(*vps).width = i32::try_from(w).unwrap();
(*vps).height = i32::try_from(h).unwrap();
(*vps).codec_id = out_video_codec.id().into();
(*vps).codec_type = ffmpeg_next::ffi::AVMediaType::AVMEDIA_TYPE_VIDEO;
};
out_video_ctx.set_parameters(video_params).unwrap();
let _out_video = output.add_stream_with(&out_video_ctx).unwrap();
let mut encoded_video = ffmpeg_next::Packet::empty();
// out_video.set_time_base(emu_time_base);
let mut out_video_enc = out_video_ctx.encoder().video().unwrap();
out_video_enc.set_format(ffmpeg_next::format::Pixel::YUV420P);
out_video_enc.set_width(u32::try_from(w).unwrap());
out_video_enc.set_height(u32::try_from(h).unwrap());
out_video_enc.set_time_base(emu_time_base);
let mut out_video_enc = out_video_enc.open().unwrap();
let out_audio_codec = ffmpeg_next::encoder::find(ffmpeg_next::codec::Id::AAC).unwrap();
let mut out_audio_ctx = ffmpeg_next::codec::context::Context::new_with_codec(out_audio_codec);
// out_audio_ctx.debug(Debug::all());
let mut audio_params = ffmpeg_next::codec::Parameters::new();
unsafe {
let aps = audio_params.as_mut_ptr();
(*aps).codec_id = out_audio_codec.id().into();
(*aps).codec_type = ffmpeg_next::ffi::AVMediaType::AVMEDIA_TYPE_AUDIO;
(*aps).frame_size = (emu.get_audio_sample_rate() / emu.get_video_fps().floor())
.to_i32()
.unwrap();
(*aps).sample_rate = emu.get_audio_sample_rate().to_i32().unwrap();
(*aps).channels = 2;
};
out_audio_ctx.set_parameters(audio_params).unwrap();
// out_audio_ctx.set_time_base(ffmpeg_next::util::rational::Rational::new(
// 1,
// emu.get_audio_sample_rate() as i32,
// ));
let _out_audio = output.add_stream_with(&out_audio_ctx).unwrap();
let mut encoded_audio = ffmpeg_next::Packet::empty();
// out_audio.set_time_base(ffmpeg_next::util::rational::Rational::new(
// 1,
// emu.get_audio_sample_rate() as i32,
// ));
let audio_time_base = ffmpeg_next::util::rational::Rational::new(
1,
emu.get_audio_sample_rate().to_i32().unwrap(),
);
let mut out_audio_enc = out_audio_ctx.encoder().audio().unwrap();
out_audio_enc.set_channels(2);
out_audio_enc.set_format(ffmpeg_next::format::Sample::F32(
ffmpeg_next::format::sample::Type::Planar,
));
out_audio_enc.set_channel_layout(ffmpeg_next::ChannelLayout::STEREO);
out_audio_enc.set_time_base(audio_time_base);
out_audio_enc.set_rate(emu.get_audio_sample_rate().to_i32().unwrap());
let mut out_audio_enc = out_audio_enc.open().unwrap();
let mut out_vframe = FFVFrame::new(
out_video_enc.format(),
out_video_enc.width(),
out_video_enc.height(),
);
let mut out_rgbframe = FFVFrame::new(
ffmpeg_next::format::Pixel::RGB24,
u32::try_from(w).unwrap(),
u32::try_from(h).unwrap(),
);
let mut out_aframe = FFAFrame::new(
out_audio_enc.format(),
out_audio_enc.frame_size() as usize,
out_audio_enc.channel_layout(),
);
let emu_time_base = Rational::new(1, emu.get_video_fps().to_i32().unwrap());
let audio_sample_rate = emu.get_audio_sample_rate().to_i32().unwrap();
let mut video_state = VideoState::new(emu_time_base, w, h, pixel_format, &mut output);
let mut audio_state = AudioState::new(audio_sample_rate, &mut output);
output.write_header().unwrap();
assert!(emu.load(&rply.initial_state));
let video_stream_time_base = output.stream(0).unwrap().time_base();
let audio_stream_time_base = output.stream(1).unwrap().time_base();
encoded_video.set_time_base(video_stream_time_base);
encoded_audio.set_time_base(audio_stream_time_base);
video_state
.encoded_video
.set_time_base(video_stream_time_base);
audio_state
.encoded_audio
.set_time_base(audio_stream_time_base);
let mut frame = Frame::default();
let mut fb = vec![0_u8; w * h * 3];
let mut converter = img_conv(
(u32::try_from(w).unwrap(), u32::try_from(h).unwrap()),
out_rgbframe.format(),
out_video_enc.format(),
)
.unwrap();
let mut audio_buf = ringbuf::LocalRb::new(out_aframe.samples() * 2 * 20);
let mut frame_audio_buf = vec![0_i16; out_aframe.samples() * 2];
let mut audio_frame = 0;
while let Ok(()) = rply
.read_frame(&mut frame)
.inspect_err(|e| println!("Err: {e}"))
{
use ffmpeg_next::util::mathematics::Rescale;
let buttons = frame_to_buttons(&frame);
emu.run(buttons);
emu.copy_framebuffer_rgb888(&mut fb).unwrap();
video_state.send_frame(&emu, rply.frame_number, &mut output);
audio_state.send_frames(&emu, &mut output);
if !frame.checkpoint_bytes.is_empty() {
assert!(emu.load(&frame.checkpoint_bytes));
}
// output one frame of video/audio, set_pts
// copy video to out_vframe
copy_video(&fb, &mut converter, &mut out_rgbframe, &mut out_vframe);
let frame_num = i64::try_from(rply.frame_number).unwrap();
let frame_pts = frame_num.rescale(emu_time_base, out_video_enc.time_base());
out_vframe.set_pts(Some(frame_pts));
out_video_enc.send_frame(&out_vframe).unwrap();
// copy audio to out_aframe, set_pts
// maybe in a loop?
#[allow(unused_must_use)]
emu.peek_audio_sample(|samples| {
audio_buf.push_slice_overwrite(samples);
while audio_buf.occupied_len() >= out_aframe.samples() * 2 {
assert_eq!(
audio_buf.pop_slice(&mut frame_audio_buf),
frame_audio_buf.len()
);
copy_audio(&frame_audio_buf, &mut out_aframe);
out_aframe.set_pts(Some(audio_frame));
audio_frame += i64::try_from(out_aframe.samples()).unwrap();
out_audio_enc.send_frame(&out_aframe).unwrap();
}
});
while out_video_enc.receive_packet(&mut encoded_video).is_ok() {
encoded_video.set_stream(0);
encoded_video.rescale_ts(out_video_enc.time_base(), video_stream_time_base);
encoded_video.write_interleaved(&mut output).unwrap();
}
while out_audio_enc.receive_packet(&mut encoded_audio).is_ok() {
encoded_audio.set_stream(1);
encoded_audio.rescale_ts(out_audio_enc.time_base(), audio_stream_time_base);
encoded_audio.write_interleaved(&mut output).unwrap();
}
if Some(rply.frame_number) == rply.header.frame_count() {
break;
}
}
while audio_buf.occupied_len() >= out_aframe.samples() {
let len = audio_buf.pop_slice(&mut frame_audio_buf);
frame_audio_buf[len..].fill(0);
out_aframe.set_pts(Some(audio_frame));
audio_frame += i64::try_from(len / 2).unwrap();
copy_audio(&frame_audio_buf, &mut out_aframe);
out_audio_enc.send_frame(&out_aframe).unwrap();
}
out_video_enc.send_eof().unwrap();
out_audio_enc.send_eof().unwrap();
while out_video_enc.receive_packet(&mut encoded_video).is_ok() {
encoded_video.set_stream(0);
encoded_video.rescale_ts(out_video_enc.time_base(), video_stream_time_base);
encoded_video.write_interleaved(&mut output).unwrap();
}
while out_audio_enc.receive_packet(&mut encoded_audio).is_ok() {
encoded_audio.set_stream(1);
encoded_audio.rescale_ts(out_audio_enc.time_base(), audio_stream_time_base);
encoded_audio.write_interleaved(&mut output).unwrap();
}
audio_state.drain(&mut output);
video_state.drain(&mut output);
output.write_trailer().unwrap();
}