diff --git a/Cargo.lock b/Cargo.lock index 12d4a98..975764b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -210,7 +210,7 @@ name = "genvideo" version = "0.1.0" dependencies = [ "ffmpeg-next", - "retro-rs 0.5.4", + "retro-rs", "ringbuf", "rply-codec", ] @@ -474,18 +474,9 @@ checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" [[package]] name = "retro-rs" -version = "0.5.3" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb116c907a72a722faf31bc45293bd4748a85109807ae5334001c7c74e188be2" -dependencies = [ - "libc", - "libloading", - "rust-libretro-sys", -] - -[[package]] -name = "retro-rs" -version = "0.5.4" +checksum = "b2dc1c5993446d4122c2f8ecf21b4e96db47678d139868848dee603511df9e1c" dependencies = [ "libc", "libloading", @@ -646,7 +637,7 @@ checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" name = "upgradev0" version = "0.1.0" dependencies = [ - "retro-rs 0.5.3", + "retro-rs", "rply-codec", ] diff --git a/genvideo/Cargo.toml b/genvideo/Cargo.toml index fbdb3a9..a16f682 100644 --- a/genvideo/Cargo.toml +++ b/genvideo/Cargo.toml @@ -5,6 +5,6 @@ edition = "2024" [dependencies] rply-codec = { path = "../codec" } -retro-rs = { version = "0.5.4", default-features=false, path = "../../retro-rs" } +retro-rs = { version = "0.5.5", default-features=false } ffmpeg-next = "8.0.0" ringbuf = "0.4.8" diff --git a/genvideo/src/main.rs b/genvideo/src/main.rs index f41e5d0..3aba515 100644 --- a/genvideo/src/main.rs +++ b/genvideo/src/main.rs @@ -1,4 +1,6 @@ +use ffmpeg_next::util::{mathematics::Rescale, rational::Rational}; use ffmpeg_next::{ + format::context::Output as FFOut, software::converter as img_conv, util::frame::{Audio as FFAFrame, Video as FFVFrame}, }; @@ -46,19 +48,227 @@ fn copy_audio(samples: &[i16], frame: &mut FFAFrame) { frame.plane_mut(1)[i] = f32::from(*r) / BOUND; } } -fn copy_video( - fb: &[u8], - converter: &mut ffmpeg_next::software::scaling::Context, - rgbframe: &mut FFVFrame, - outframe: &mut FFVFrame, -) { - rgbframe.data_mut(0).copy_from_slice(fb); - converter.run(rgbframe, outframe).unwrap(); +struct VideoState { + out_video_enc: ffmpeg_next::encoder::video::Encoder, + out_vframe: FFVFrame, + out_rgbframe: FFVFrame, + encoded_video: ffmpeg_next::Packet, + converter: ffmpeg_next::software::scaling::Context, + emu_time_base: Rational, + native_pixel_format: bool, +} + +impl VideoState { + fn new( + emu_time_base: Rational, + w: usize, + h: usize, + pixel_format: retro_rs::libretro::retro_pixel_format, + output: &mut FFOut, + ) -> Self { + let out_video_codec = ffmpeg_next::encoder::find(ffmpeg_next::codec::Id::H264).unwrap(); + let mut out_video_ctx = + ffmpeg_next::codec::context::Context::new_with_codec(out_video_codec); + // out_video_ctx.set_time_base(emu_time_base); + let mut video_params = ffmpeg_next::codec::Parameters::new(); + unsafe { + let vps = video_params.as_mut_ptr(); + (*vps).width = i32::try_from(w).unwrap(); + (*vps).height = i32::try_from(h).unwrap(); + (*vps).codec_id = out_video_codec.id().into(); + (*vps).codec_type = ffmpeg_next::ffi::AVMediaType::AVMEDIA_TYPE_VIDEO; + }; + out_video_ctx.set_parameters(video_params).unwrap(); + let _out_video = output.add_stream_with(&out_video_ctx).unwrap(); + let encoded_video = ffmpeg_next::Packet::empty(); + // out_video.set_time_base(emu_time_base); + let mut out_video_enc = out_video_ctx.encoder().video().unwrap(); + out_video_enc.set_format(ffmpeg_next::format::Pixel::YUV420P); + out_video_enc.set_width(u32::try_from(w).unwrap()); + out_video_enc.set_height(u32::try_from(h).unwrap()); + out_video_enc.set_time_base(emu_time_base); + let out_video_enc = out_video_enc.open().unwrap(); + let out_vframe = FFVFrame::new( + out_video_enc.format(), + out_video_enc.width(), + out_video_enc.height(), + ); + /* TODO: Consider using the pixel format of the emulator here and avoid copying as rgb888 */ + let (copy_format, is_native) = match pixel_format { + retro_rs::libretro::retro_pixel_format::RETRO_PIXEL_FORMAT_0RGB1555 => { + (ffmpeg_next::format::Pixel::RGB555, true) + } + retro_rs::libretro::retro_pixel_format::RETRO_PIXEL_FORMAT_XRGB8888 => { + (ffmpeg_next::format::Pixel::ZRGB, true) + } + retro_rs::libretro::retro_pixel_format::RETRO_PIXEL_FORMAT_RGB565 => { + (ffmpeg_next::format::Pixel::RGB565, true) + } + _other => (ffmpeg_next::format::Pixel::RGB24, false), + }; + let out_rgbframe = FFVFrame::new( + copy_format, + u32::try_from(w).unwrap(), + u32::try_from(h).unwrap(), + ); + + let converter = img_conv( + (u32::try_from(w).unwrap(), u32::try_from(h).unwrap()), + out_rgbframe.format(), + out_video_enc.format(), + ) + .unwrap(); + Self { + out_video_enc, + out_vframe, + out_rgbframe, + encoded_video, + converter, + emu_time_base, + native_pixel_format: is_native, + } + } + fn writeout(&mut self, output: &mut FFOut) { + let output_time_base = output.stream(0).unwrap().time_base(); + while self + .out_video_enc + .receive_packet(&mut self.encoded_video) + .is_ok() + { + self.encoded_video.set_stream(0); + self.encoded_video + .rescale_ts(self.out_video_enc.time_base(), output_time_base); + self.encoded_video.write_interleaved(output).unwrap(); + } + } + fn send_frame(&mut self, emu: &Emulator, frame_num: u64, output: &mut FFOut) { + // output one frame of video/audio, set_pts + // copy video to out_vframe + if self.native_pixel_format { + emu.peek_framebuffer(|fb| { + self.out_rgbframe.data_mut(0).copy_from_slice(fb); + }) + .unwrap(); + } else { + emu.copy_framebuffer_rgb888(self.out_rgbframe.data_mut(0)) + .unwrap(); + } + self.converter + .run(&self.out_rgbframe, &mut self.out_vframe) + .unwrap(); + let frame_num = i64::try_from(frame_num).unwrap(); + let frame_pts = frame_num.rescale(self.emu_time_base, self.out_video_enc.time_base()); + self.out_vframe.set_pts(Some(frame_pts)); + self.out_video_enc.send_frame(&self.out_vframe).unwrap(); + self.writeout(output); + } + fn drain(&mut self, output: &mut FFOut) { + self.out_video_enc.send_eof().unwrap(); + self.writeout(output); + } +} + +struct AudioState { + out_audio_enc: ffmpeg_next::encoder::audio::Encoder, + out_aframe: FFAFrame, + encoded_audio: ffmpeg_next::Packet, + audio_buf: ringbuf::LocalRb>, + frame_audio_buf: Vec, + audio_frame: i64, +} + +impl AudioState { + fn new(audio_sample_rate: i32, output: &mut FFOut) -> Self { + let out_audio_codec = ffmpeg_next::encoder::find(ffmpeg_next::codec::Id::AAC).unwrap(); + let mut out_audio_ctx = + ffmpeg_next::codec::context::Context::new_with_codec(out_audio_codec); + // out_audio_ctx.debug(Debug::all()); + let mut audio_params = ffmpeg_next::codec::Parameters::new(); + unsafe { + let aps = audio_params.as_mut_ptr(); + (*aps).codec_id = out_audio_codec.id().into(); + (*aps).codec_type = ffmpeg_next::ffi::AVMediaType::AVMEDIA_TYPE_AUDIO; + (*aps).sample_rate = audio_sample_rate; + (*aps).frame_size = 1024 * 2 * 4; + (*aps).channels = 2; + }; + out_audio_ctx.set_parameters(audio_params).unwrap(); + let _out_audio = output.add_stream_with(&out_audio_ctx).unwrap(); + let encoded_audio = ffmpeg_next::Packet::empty(); + let audio_time_base = Rational::new(1, audio_sample_rate); + let mut out_audio_enc = out_audio_ctx.encoder().audio().unwrap(); + out_audio_enc.set_channels(2); + out_audio_enc.set_format(ffmpeg_next::format::Sample::F32( + ffmpeg_next::format::sample::Type::Planar, + )); + out_audio_enc.set_channel_layout(ffmpeg_next::ChannelLayout::STEREO); + out_audio_enc.set_time_base(audio_time_base); + out_audio_enc.set_rate(audio_sample_rate); + let out_audio_enc = out_audio_enc.open().unwrap(); + let out_aframe = FFAFrame::new( + out_audio_enc.format(), + out_audio_enc.frame_size() as usize, + out_audio_enc.channel_layout(), + ); + let audio_buf = ringbuf::LocalRb::new(out_aframe.samples() * 2 * 20); + let frame_audio_buf = vec![0_i16; out_aframe.samples() * 2]; + + Self { + out_audio_enc, + out_aframe, + encoded_audio, + audio_buf, + frame_audio_buf, + audio_frame: 0, + } + } + fn writeout(&mut self, output: &mut FFOut) { + let output_time_base = output.stream(1).unwrap().time_base(); + while self + .out_audio_enc + .receive_packet(&mut self.encoded_audio) + .is_ok() + { + self.encoded_audio.set_stream(1); + self.encoded_audio + .rescale_ts(self.out_audio_enc.time_base(), output_time_base); + self.encoded_audio.write_interleaved(output).unwrap(); + } + } + fn send_frames(&mut self, emu: &Emulator, output: &mut FFOut) { + #[allow(unused_must_use)] + emu.peek_audio_sample(|samples| { + self.audio_buf.push_slice_overwrite(samples); + while self.audio_buf.occupied_len() >= self.out_aframe.samples() * 2 { + assert_eq!( + self.audio_buf.pop_slice(&mut self.frame_audio_buf), + self.frame_audio_buf.len() + ); + copy_audio(&self.frame_audio_buf, &mut self.out_aframe); + self.out_aframe.set_pts(Some(self.audio_frame)); + self.audio_frame += i64::try_from(self.out_aframe.samples()).unwrap(); + self.out_audio_enc.send_frame(&self.out_aframe).unwrap(); + } + }); + self.writeout(output); + } + fn drain(&mut self, output: &mut FFOut) { + while self.audio_buf.occupied_len() >= self.out_aframe.samples() { + let len = self.audio_buf.pop_slice(&mut self.frame_audio_buf); + self.frame_audio_buf[len..].fill(0); + self.out_aframe.set_pts(Some(self.audio_frame)); + self.audio_frame += i64::try_from(len / 2).unwrap(); + copy_audio(&self.frame_audio_buf, &mut self.out_aframe); + self.out_audio_enc.send_frame(&self.out_aframe).unwrap(); + } + self.out_audio_enc.send_eof().unwrap(); + self.writeout(output); + } } fn main() { ffmpeg_next::init().unwrap(); - ffmpeg_next::log::set_level(ffmpeg_next::log::Level::Trace); + ffmpeg_next::log::set_level(ffmpeg_next::log::Level::Warning); let args: Vec<_> = std::env::args().collect(); let file = std::fs::File::open(args.get(1).unwrap_or(&"examples/bobl.replay".to_string())).unwrap(); @@ -79,174 +289,44 @@ fn main() { } // run emu a tick to make sure we have right frame sizes, etc emu.run([retro_rs::Buttons::default(); 2]); - let (w, h) = emu.framebuffer_size(); + let pixel_format = emu.pixel_format(); + assert!(emu.load(&rply.initial_state)); let mut output = ffmpeg_next::format::output(&outfile).unwrap(); - let emu_time_base = - ffmpeg_next::util::rational::Rational::new(1, emu.get_video_fps().to_i32().unwrap()); - let out_video_codec = ffmpeg_next::encoder::find(ffmpeg_next::codec::Id::H264).unwrap(); - let mut out_video_ctx = ffmpeg_next::codec::context::Context::new_with_codec(out_video_codec); - // out_video_ctx.set_time_base(emu_time_base); - let mut video_params = ffmpeg_next::codec::Parameters::new(); - unsafe { - let vps = video_params.as_mut_ptr(); - (*vps).width = i32::try_from(w).unwrap(); - (*vps).height = i32::try_from(h).unwrap(); - (*vps).codec_id = out_video_codec.id().into(); - (*vps).codec_type = ffmpeg_next::ffi::AVMediaType::AVMEDIA_TYPE_VIDEO; - }; - out_video_ctx.set_parameters(video_params).unwrap(); - let _out_video = output.add_stream_with(&out_video_ctx).unwrap(); - let mut encoded_video = ffmpeg_next::Packet::empty(); - // out_video.set_time_base(emu_time_base); - let mut out_video_enc = out_video_ctx.encoder().video().unwrap(); - out_video_enc.set_format(ffmpeg_next::format::Pixel::YUV420P); - out_video_enc.set_width(u32::try_from(w).unwrap()); - out_video_enc.set_height(u32::try_from(h).unwrap()); - out_video_enc.set_time_base(emu_time_base); - let mut out_video_enc = out_video_enc.open().unwrap(); - let out_audio_codec = ffmpeg_next::encoder::find(ffmpeg_next::codec::Id::AAC).unwrap(); - let mut out_audio_ctx = ffmpeg_next::codec::context::Context::new_with_codec(out_audio_codec); - // out_audio_ctx.debug(Debug::all()); - let mut audio_params = ffmpeg_next::codec::Parameters::new(); - unsafe { - let aps = audio_params.as_mut_ptr(); - (*aps).codec_id = out_audio_codec.id().into(); - (*aps).codec_type = ffmpeg_next::ffi::AVMediaType::AVMEDIA_TYPE_AUDIO; - (*aps).frame_size = (emu.get_audio_sample_rate() / emu.get_video_fps().floor()) - .to_i32() - .unwrap(); - (*aps).sample_rate = emu.get_audio_sample_rate().to_i32().unwrap(); - (*aps).channels = 2; - }; - out_audio_ctx.set_parameters(audio_params).unwrap(); - // out_audio_ctx.set_time_base(ffmpeg_next::util::rational::Rational::new( - // 1, - // emu.get_audio_sample_rate() as i32, - // )); - let _out_audio = output.add_stream_with(&out_audio_ctx).unwrap(); - let mut encoded_audio = ffmpeg_next::Packet::empty(); - // out_audio.set_time_base(ffmpeg_next::util::rational::Rational::new( - // 1, - // emu.get_audio_sample_rate() as i32, - // )); - let audio_time_base = ffmpeg_next::util::rational::Rational::new( - 1, - emu.get_audio_sample_rate().to_i32().unwrap(), - ); - let mut out_audio_enc = out_audio_ctx.encoder().audio().unwrap(); - out_audio_enc.set_channels(2); - out_audio_enc.set_format(ffmpeg_next::format::Sample::F32( - ffmpeg_next::format::sample::Type::Planar, - )); - out_audio_enc.set_channel_layout(ffmpeg_next::ChannelLayout::STEREO); - out_audio_enc.set_time_base(audio_time_base); - out_audio_enc.set_rate(emu.get_audio_sample_rate().to_i32().unwrap()); - let mut out_audio_enc = out_audio_enc.open().unwrap(); - let mut out_vframe = FFVFrame::new( - out_video_enc.format(), - out_video_enc.width(), - out_video_enc.height(), - ); - let mut out_rgbframe = FFVFrame::new( - ffmpeg_next::format::Pixel::RGB24, - u32::try_from(w).unwrap(), - u32::try_from(h).unwrap(), - ); - let mut out_aframe = FFAFrame::new( - out_audio_enc.format(), - out_audio_enc.frame_size() as usize, - out_audio_enc.channel_layout(), - ); + let emu_time_base = Rational::new(1, emu.get_video_fps().to_i32().unwrap()); + let audio_sample_rate = emu.get_audio_sample_rate().to_i32().unwrap(); + let mut video_state = VideoState::new(emu_time_base, w, h, pixel_format, &mut output); + let mut audio_state = AudioState::new(audio_sample_rate, &mut output); output.write_header().unwrap(); - assert!(emu.load(&rply.initial_state)); let video_stream_time_base = output.stream(0).unwrap().time_base(); let audio_stream_time_base = output.stream(1).unwrap().time_base(); - encoded_video.set_time_base(video_stream_time_base); - encoded_audio.set_time_base(audio_stream_time_base); + video_state + .encoded_video + .set_time_base(video_stream_time_base); + audio_state + .encoded_audio + .set_time_base(audio_stream_time_base); let mut frame = Frame::default(); - let mut fb = vec![0_u8; w * h * 3]; - let mut converter = img_conv( - (u32::try_from(w).unwrap(), u32::try_from(h).unwrap()), - out_rgbframe.format(), - out_video_enc.format(), - ) - .unwrap(); - let mut audio_buf = ringbuf::LocalRb::new(out_aframe.samples() * 2 * 20); - let mut frame_audio_buf = vec![0_i16; out_aframe.samples() * 2]; - let mut audio_frame = 0; while let Ok(()) = rply .read_frame(&mut frame) .inspect_err(|e| println!("Err: {e}")) { - use ffmpeg_next::util::mathematics::Rescale; let buttons = frame_to_buttons(&frame); emu.run(buttons); - emu.copy_framebuffer_rgb888(&mut fb).unwrap(); + video_state.send_frame(&emu, rply.frame_number, &mut output); + audio_state.send_frames(&emu, &mut output); if !frame.checkpoint_bytes.is_empty() { assert!(emu.load(&frame.checkpoint_bytes)); } - // output one frame of video/audio, set_pts - // copy video to out_vframe - copy_video(&fb, &mut converter, &mut out_rgbframe, &mut out_vframe); - let frame_num = i64::try_from(rply.frame_number).unwrap(); - let frame_pts = frame_num.rescale(emu_time_base, out_video_enc.time_base()); - out_vframe.set_pts(Some(frame_pts)); - out_video_enc.send_frame(&out_vframe).unwrap(); - // copy audio to out_aframe, set_pts - // maybe in a loop? - #[allow(unused_must_use)] - emu.peek_audio_sample(|samples| { - audio_buf.push_slice_overwrite(samples); - while audio_buf.occupied_len() >= out_aframe.samples() * 2 { - assert_eq!( - audio_buf.pop_slice(&mut frame_audio_buf), - frame_audio_buf.len() - ); - copy_audio(&frame_audio_buf, &mut out_aframe); - out_aframe.set_pts(Some(audio_frame)); - audio_frame += i64::try_from(out_aframe.samples()).unwrap(); - out_audio_enc.send_frame(&out_aframe).unwrap(); - } - }); - while out_video_enc.receive_packet(&mut encoded_video).is_ok() { - encoded_video.set_stream(0); - encoded_video.rescale_ts(out_video_enc.time_base(), video_stream_time_base); - encoded_video.write_interleaved(&mut output).unwrap(); - } - while out_audio_enc.receive_packet(&mut encoded_audio).is_ok() { - encoded_audio.set_stream(1); - encoded_audio.rescale_ts(out_audio_enc.time_base(), audio_stream_time_base); - encoded_audio.write_interleaved(&mut output).unwrap(); - } + if Some(rply.frame_number) == rply.header.frame_count() { break; } } - while audio_buf.occupied_len() >= out_aframe.samples() { - let len = audio_buf.pop_slice(&mut frame_audio_buf); - frame_audio_buf[len..].fill(0); - out_aframe.set_pts(Some(audio_frame)); - audio_frame += i64::try_from(len / 2).unwrap(); - copy_audio(&frame_audio_buf, &mut out_aframe); - out_audio_enc.send_frame(&out_aframe).unwrap(); - } - - out_video_enc.send_eof().unwrap(); - out_audio_enc.send_eof().unwrap(); - - while out_video_enc.receive_packet(&mut encoded_video).is_ok() { - encoded_video.set_stream(0); - encoded_video.rescale_ts(out_video_enc.time_base(), video_stream_time_base); - encoded_video.write_interleaved(&mut output).unwrap(); - } - while out_audio_enc.receive_packet(&mut encoded_audio).is_ok() { - encoded_audio.set_stream(1); - encoded_audio.rescale_ts(out_audio_enc.time_base(), audio_stream_time_base); - encoded_audio.write_interleaved(&mut output).unwrap(); - } + audio_state.drain(&mut output); + video_state.drain(&mut output); output.write_trailer().unwrap(); }