diff --git a/Cargo.lock b/Cargo.lock index 33ac517..12d4a98 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -134,6 +134,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "dump" version = "0.1.0" @@ -205,6 +211,7 @@ version = "0.1.0" dependencies = [ "ffmpeg-next", "retro-rs 0.5.4", + "ringbuf", "rply-codec", ] @@ -390,6 +397,21 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "portable-atomic" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + [[package]] name = "proc-macro2" version = "1.0.103" @@ -470,6 +492,17 @@ dependencies = [ "rust-libretro-sys", ] +[[package]] +name = "ringbuf" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe47b720588c8702e34b5979cb3271a8b1842c7cb6f57408efa70c779363488c" +dependencies = [ + "crossbeam-utils", + "portable-atomic", + "portable-atomic-util", +] + [[package]] name = "rmp" version = "0.8.14" diff --git a/genvideo/Cargo.toml b/genvideo/Cargo.toml index c015540..fbdb3a9 100644 --- a/genvideo/Cargo.toml +++ b/genvideo/Cargo.toml @@ -6,4 +6,5 @@ edition = "2024" [dependencies] rply-codec = { path = "../codec" } retro-rs = { version = "0.5.4", default-features=false, path = "../../retro-rs" } -ffmpeg-next = "8.0.0" \ No newline at end of file +ffmpeg-next = "8.0.0" +ringbuf = "0.4.8" diff --git a/genvideo/src/main.rs b/genvideo/src/main.rs index 1ca140c..af2abfa 100644 --- a/genvideo/src/main.rs +++ b/genvideo/src/main.rs @@ -1,42 +1,27 @@ use ffmpeg_next::{ - codec::Debug, + software::converter as img_conv, util::frame::{Audio as FFAFrame, Video as FFVFrame}, }; use retro_rs::Emulator; +use ringbuf::traits::{Consumer, Observer, RingBuffer}; use rply_codec::{Frame, decode}; use std::path::Path; fn copy_audio(samples: &[i16], frame: &mut FFAFrame) { - println!( - "Copy {} samples into {}", - samples.len() / 2, - frame.samples() - ); for (i, pair) in samples.chunks_exact(2).enumerate() { - let [l, r] = pair else { - unreachable!(); - }; - frame.plane_mut(0)[i] = f32::from(*l) / 65535.0; - frame.plane_mut(1)[i] = f32::from(*r) / 65535.0; + let [l, r] = pair else { unreachable!() }; + frame.plane_mut(0)[i] = *l as f32 / i16::MAX as f32; + frame.plane_mut(1)[i] = *r as f32 / i16::MAX as f32; } } -fn copy_video(fb: &[u8], frame: &mut FFVFrame) { - let w = i32::try_from(frame.width()).unwrap(); - let h = i32::try_from(frame.height()).unwrap(); - let fmt: ffmpeg_next::ffi::AVPixelFormat = frame.format().into(); - unsafe { - let frame_ptr = frame.as_mut_ptr(); - let ret = ffmpeg_next::ffi::av_image_fill_arrays( - (*frame_ptr).data.as_mut_ptr(), - (*frame_ptr).linesize.as_mut_ptr(), - fb.as_ptr(), - fmt, - w, - h, - 1, - ); - assert!(ret > 0, "av_image_fill_arrays returned {ret}"); - } +fn copy_video( + fb: &[u8], + converter: &mut ffmpeg_next::software::scaling::Context, + rgbframe: &mut FFVFrame, + outframe: &mut FFVFrame, +) { + rgbframe.data_mut(0).copy_from_slice(fb); + converter.run(rgbframe, outframe).unwrap(); } fn main() { @@ -69,50 +54,51 @@ fn main() { let emu_time_base = ffmpeg_next::util::rational::Rational::new(1, emu.get_video_fps() as i32); let out_video_codec = ffmpeg_next::encoder::find(ffmpeg_next::codec::Id::H264).unwrap(); let mut out_video_ctx = ffmpeg_next::codec::context::Context::new_with_codec(out_video_codec); + // out_video_ctx.set_time_base(emu_time_base); let mut video_params = ffmpeg_next::codec::Parameters::new(); unsafe { let vps = video_params.as_mut_ptr(); (*vps).width = i32::try_from(w).unwrap(); (*vps).height = i32::try_from(h).unwrap(); (*vps).codec_id = out_video_codec.id().into(); - (*vps).framerate = ffmpeg_next::util::rational::Rational::from(emu.get_video_fps()).into(); + (*vps).codec_type = ffmpeg_next::ffi::AVMediaType::AVMEDIA_TYPE_VIDEO; }; out_video_ctx.set_parameters(video_params).unwrap(); - out_video_ctx.set_time_base(emu_time_base); - out_video_ctx.debug(Debug::all()); - dbg!(out_video_ctx.codec().unwrap().id()); let mut out_video = output.add_stream_with(&out_video_ctx).unwrap(); - out_video.set_time_base(emu_time_base); - let video_time_base = out_video.time_base(); + let mut encoded_video = ffmpeg_next::Packet::empty(); + // out_video.set_time_base(emu_time_base); let mut out_video_enc = out_video_ctx.encoder().video().unwrap(); - out_video_enc.debug(Debug::all()); out_video_enc.set_format(ffmpeg_next::format::Pixel::YUV420P); out_video_enc.set_width(u32::try_from(w).unwrap()); out_video_enc.set_height(u32::try_from(h).unwrap()); - out_video_enc.set_time_base(video_time_base); + out_video_enc.set_time_base(emu_time_base); let mut out_video_enc = out_video_enc.open().unwrap(); let out_audio_codec = ffmpeg_next::encoder::find(ffmpeg_next::codec::Id::AAC).unwrap(); let mut out_audio_ctx = ffmpeg_next::codec::context::Context::new_with_codec(out_audio_codec); - out_audio_ctx.debug(Debug::all()); + // out_audio_ctx.debug(Debug::all()); let mut audio_params = ffmpeg_next::codec::Parameters::new(); unsafe { let aps = audio_params.as_mut_ptr(); (*aps).codec_id = out_audio_codec.id().into(); + (*aps).codec_type = ffmpeg_next::ffi::AVMediaType::AVMEDIA_TYPE_AUDIO; + (*aps).frame_size = (emu.get_audio_sample_rate() / emu.get_video_fps().floor()) as i32; (*aps).sample_rate = emu.get_audio_sample_rate() as i32; (*aps).channels = 2; }; out_audio_ctx.set_parameters(audio_params).unwrap(); - out_audio_ctx.set_time_base(ffmpeg_next::util::rational::Rational::new( - 1, - emu.get_audio_sample_rate() as i32, - )); + // out_audio_ctx.set_time_base(ffmpeg_next::util::rational::Rational::new( + // 1, + // emu.get_audio_sample_rate() as i32, + // )); let mut out_audio = output.add_stream_with(&out_audio_ctx).unwrap(); - out_audio.set_time_base(ffmpeg_next::util::rational::Rational::new( - 1, - emu.get_audio_sample_rate() as i32, - )); - let audio_time_base = out_audio.time_base(); - dbg!(audio_time_base); + let mut encoded_audio = ffmpeg_next::Packet::empty(); + // out_audio.set_time_base(ffmpeg_next::util::rational::Rational::new( + // 1, + // emu.get_audio_sample_rate() as i32, + // )); + let audio_time_base = + ffmpeg_next::util::rational::Rational::new(1, emu.get_audio_sample_rate() as i32); + // dbg!(audio_time_base); let mut out_audio_enc = out_audio_ctx.encoder().audio().unwrap(); out_audio_enc.set_channels(2); out_audio_enc.set_format(ffmpeg_next::format::Sample::F32( @@ -127,86 +113,130 @@ fn main() { out_video_enc.width(), out_video_enc.height(), ); + let mut out_rgbframe = FFVFrame::new( + ffmpeg_next::format::Pixel::RGB24, + u32::try_from(w).unwrap(), + u32::try_from(h).unwrap(), + ); let mut out_aframe = FFAFrame::new( out_audio_enc.format(), out_audio_enc.frame_size() as usize, out_audio_enc.channel_layout(), ); - dbg!( - out_audio_enc.channels(), + let mut out_i16frame = FFAFrame::new( + ffmpeg_next::format::Sample::I16(ffmpeg_next::format::sample::Type::Packed), + out_audio_enc.frame_size() as usize, out_audio_enc.channel_layout(), - out_audio_enc.format() ); - dbg!(out_aframe.samples()); - dbg!(out_aframe.data(0).len(), out_aframe.data(1).len()); + // dbg!( + // out_audio_enc.channels(), + // out_audio_enc.channel_layout(), + // out_audio_enc.format() + // ); + // dbg!(out_aframe.samples()); + // dbg!(out_aframe.data(0).len(), out_aframe.data(1).len()); output.write_header().unwrap(); assert!(emu.load(&rply.initial_state)); + let video_stream_time_base = output.stream(0).unwrap().time_base(); + let audio_stream_time_base = output.stream(1).unwrap().time_base(); + encoded_video.set_time_base(video_stream_time_base); + encoded_audio.set_time_base(audio_stream_time_base); + let mut frame = Frame::default(); let mut fb = vec![0_u8; w * h * 3]; - + let mut converter = img_conv( + (u32::try_from(w).unwrap(), u32::try_from(h).unwrap()), + out_rgbframe.format(), + out_video_enc.format(), + ) + .unwrap(); + let mut audio_buf = ringbuf::LocalRb::new(out_aframe.samples() * 2 * 4); + let mut frame_audio_buf = vec![0_i16; out_aframe.samples() * 2]; + let mut audio_frame = 0; while let Ok(()) = rply .read_frame(&mut frame) .inspect_err(|e| println!("Err: {e}")) { use ffmpeg_next::util::mathematics::Rescale; - //println!("FRAME"); let buttons = frame_to_buttons(&frame); emu.run(buttons); emu.copy_framebuffer_rgb888(&mut fb).unwrap(); + if !frame.checkpoint_bytes.is_empty() { + // println!("Load CP at {frame_num}"); + assert!(emu.load(&frame.checkpoint_bytes)); + } // output one frame of video/audio, set_pts // copy video to out_vframe - copy_video(&fb, &mut out_vframe); + copy_video(&fb, &mut converter, &mut out_rgbframe, &mut out_vframe); let frame_num = i64::try_from(rply.frame_number).unwrap(); - out_vframe.set_pts(Some( - frame_num.rescale(emu_time_base, out_video_enc.time_base()), - )); + let frame_pts = frame_num.rescale(emu_time_base, out_video_enc.time_base()); + // let mut rgb = image::RgbImage::new(w as u32, h as u32); + // rgb.clone_from_slice(&fb); + // rgb.save(format!("test/{frame_num}.png")).unwrap(); + out_vframe.set_pts(Some(frame_pts)); out_video_enc.send_frame(&out_vframe).unwrap(); // copy audio to out_aframe, set_pts // maybe in a loop? #[allow(unused_must_use)] emu.peek_audio_sample(|samples| { - copy_audio(samples, &mut out_aframe); + audio_buf.push_slice_overwrite(samples); + while audio_buf.occupied_len() >= out_aframe.samples() { + enc_audio = true; + audio_buf.pop_slice(&mut frame_audio_buf); + copy_audio(&frame_audio_buf, &mut out_aframe); + out_aframe.set_pts(Some(audio_frame)); + audio_frame += out_aframe.samples(); + out_audio_enc.send_frame(&out_aframe).unwrap(); + } }); - out_aframe.set_pts(Some( - frame_num.rescale(emu_time_base, out_audio_enc.time_base()), - )); - println!( - "vtime {} atime {}", - out_vframe.pts().unwrap() as f64 / 60.0, - out_aframe.pts().unwrap() as f64 / 48000.0 - ); - out_audio_enc.send_frame(&out_aframe).unwrap(); - let mut encoded = ffmpeg_next::Packet::empty(); - while out_video_enc.receive_packet(&mut encoded).is_ok() { - encoded.set_stream(0); - encoded.rescale_ts(out_video_enc.time_base(), video_time_base); - encoded.write_interleaved(&mut output).unwrap(); + // println!( + // "vtime {} atime {}", + // out_vframe.pts().unwrap() as f64 / 60.0, + // out_aframe.pts().unwrap() as f64 / 48000.0 + // ); + while out_video_enc.receive_packet(&mut encoded_video).is_ok() { + encoded_video.set_stream(0); + encoded_video.rescale_ts(out_video_enc.time_base(), video_stream_time_base); + encoded_video.write_interleaved(&mut output).unwrap(); } - while out_audio_enc.receive_packet(&mut encoded).is_ok() { - encoded.set_stream(1); - encoded.rescale_ts(out_audio_enc.time_base(), audio_time_base); - encoded.write_interleaved(&mut output).unwrap(); - } - if !frame.checkpoint_bytes.is_empty() { - assert!(emu.load(&frame.checkpoint_bytes)); + while out_audio_enc.receive_packet(&mut encoded_audio).is_ok() { + encoded_audio.set_stream(1); + encoded_audio.rescale_ts(out_audio_enc.time_base(), audio_stream_time_base); + encoded_audio.write_interleaved(&mut output).unwrap(); } if Some(rply.frame_number) == rply.header.frame_count() { break; } } + while audio_buf.occupied_len() >= out_aframe.samples() { + let len = audio_buf.pop_slice(&mut frame_audio_buf); + frame_audio_buf[len..].fill(0); + out_aframe.set_pts(Some(audio_frame)); + out_aframe.set_samples(len / 2); + audio_frame += out_aframe.samples(); + copy_audio(&frame_audio_buf, &mut out_aframe); + out_audio_enc.send_frame(&out_aframe).unwrap(); + } + while out_audio_enc.receive_packet(&mut encoded_audio).is_ok() { + encoded_audio.set_stream(1); + encoded_audio.rescale_ts(out_audio_enc.time_base(), audio_stream_time_base); + encoded_audio.write_interleaved(&mut output).unwrap(); + } + out_video_enc.send_eof().unwrap(); out_audio_enc.send_eof().unwrap(); output.write_trailer().unwrap(); + dbg!(output.stream(0).unwrap().time_base()); } fn frame_to_buttons(frame: &Frame) -> [retro_rs::Buttons; 2] { use retro_rs::Buttons; - let mut buttons = [Buttons::default(); 2]; + let mut buttons = [0_i16; 2]; for inp in &frame.input_events { let port = usize::from(inp.port); if port < buttons.len() && inp.device == 1 { - buttons[port] = Buttons::from(inp.val); + buttons[port] |= inp.val; } } - buttons + [Buttons::from(buttons[0]), Buttons::from(buttons[1])] }