update for ffmpeg api

Cargo.lock

@@ -60,7 +60,7 @@ dependencies = [
  "regex",
  "rustc-hash 2.1.1",
  "shlex",
- "syn 2.0.108",
+ "syn 2.0.109",
 ]
 
 [[package]]
@@ -89,9 +89,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
 
 [[package]]
 name = "cc"
-version = "1.2.44"
+version = "1.2.45"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37521ac7aabe3d13122dc382493e20c9416f299d2ccd5b3a5340a2570cdeb0f3"
+checksum = "35900b6c8d709fb1d854671ae27aeaa9eec2f8b01b364e1619a40da3e6fe2afe"
 dependencies = [
  "find-msvc-tools",
  "jobserver",
@@ -210,7 +210,7 @@ name = "genvideo"
 version = "0.1.0"
 dependencies = [
  "ffmpeg-next",
- "retro-rs 0.5.4",
+ "retro-rs",
  "ringbuf",
  "rply-codec",
 ]
@@ -423,9 +423,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.41"
+version = "1.0.42"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1"
+checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f"
 dependencies = [
  "proc-macro2",
 ]
@@ -474,18 +474,9 @@ checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58"
 
 [[package]]
 name = "retro-rs"
-version = "0.5.3"
+version = "0.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb116c907a72a722faf31bc45293bd4748a85109807ae5334001c7c74e188be2"
+checksum = "4449b8e5e3de1149160ff8c2cd4ad39e5b6f4cf320abc4b917879bf6c7fbd382"
-dependencies = [
- "libc",
- "libloading",
- "rust-libretro-sys",
-]
-
-[[package]]
-name = "retro-rs"
-version = "0.5.4"
 dependencies = [
  "libc",
  "libloading",
@@ -607,9 +598,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.108"
+version = "2.0.109"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da58917d35242480a05c2897064da0a80589a2a0476c9a3f2fdc83b53502e917"
+checksum = "2f17c7e013e88258aa9543dcbe81aca68a667a9ac37cd69c9fbc07858bfe0e2f"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -633,7 +624,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.108",
+ "syn 2.0.109",
 ]
 
 [[package]]
@@ -646,7 +637,7 @@ checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5"
 name = "upgradev0"
 version = "0.1.0"
 dependencies = [
- "retro-rs 0.5.3",
+ "retro-rs",
  "rply-codec",
 ]
 

Cargo.toml

@@ -1,3 +1,6 @@
 [workspace]
 resolver = "3"
-members = ["codec", "dump", "reencode", "upgradev0", "genvideo"]
+members = ["codec", "dump", "reencode", "upgradev0", "genvideo"]
+
+[profile.release]
+debug = true

genvideo/Cargo.toml

@@ -5,6 +5,6 @@ edition = "2024"
 
 [dependencies]
 rply-codec = { path = "../codec" }
-retro-rs = { version = "0.5.4", default-features=false, path = "../../retro-rs" }
+retro-rs = { version = "0.5.6", default-features=false }
 ffmpeg-next = "8.0.0"
 ringbuf = "0.4.8"

genvideo/src/main.rs

@@ -1,4 +1,7 @@
+use ffmpeg_next::ChannelLayout;
+use ffmpeg_next::util::{mathematics::Rescale, rational::Rational};
 use ffmpeg_next::{
+    format::context::Output as FFOut,
     software::converter as img_conv,
     util::frame::{Audio as FFAFrame, Video as FFVFrame},
 };
@@ -38,36 +41,290 @@ impl ToI32 for f64 {
     }
 }
 
-fn copy_audio(samples: &[i16], frame: &mut FFAFrame) {
+struct VideoState {
-    const BOUND: f32 = i16::MAX as f32;
+    out_video_enc: ffmpeg_next::encoder::video::Encoder,
-    for (i, pair) in samples.chunks_exact(2).enumerate() {
+    out_vframe: FFVFrame,
-        let [l, r] = pair else { unreachable!() };
+    out_rgbframe: FFVFrame,
-        frame.plane_mut(0)[i] = f32::from(*l) / BOUND;
+    encoded_video: ffmpeg_next::Packet,
-        frame.plane_mut(1)[i] = f32::from(*r) / BOUND;
+    converter: ffmpeg_next::software::scaling::Context,
+    emu_time_base: Rational,
+    native_pixel_format: bool,
+    stride: usize,
+}
+
+impl VideoState {
+    fn new(
+        emu_time_base: Rational,
+        aspect_ratio: Rational,
+        w: usize,
+        h: usize,
+        pixel_format: retro_rs::libretro::retro_pixel_format,
+        output: &mut FFOut,
+    ) -> Self {
+        let out_video_codec = ffmpeg_next::encoder::find(ffmpeg_next::codec::Id::H264).unwrap();
+        let mut out_video_ctx =
+            ffmpeg_next::codec::context::Context::new_with_codec(out_video_codec);
+        // out_video_ctx.set_time_base(emu_time_base);
+        let mut video_params = ffmpeg_next::codec::Parameters::new();
+        unsafe {
+            let vps = video_params.as_mut_ptr();
+            (*vps).width = i32::try_from(w).unwrap();
+            (*vps).height = i32::try_from(h).unwrap();
+            (*vps).codec_id = out_video_codec.id().into();
+            (*vps).codec_type = ffmpeg_next::ffi::AVMediaType::AVMEDIA_TYPE_VIDEO;
+            (*vps).sample_aspect_ratio = aspect_ratio.into();
+        };
+        out_video_ctx.set_parameters(video_params).unwrap();
+        let _out_video = output.add_stream_with(&out_video_ctx).unwrap();
+        let encoded_video = ffmpeg_next::Packet::empty();
+        // out_video.set_time_base(emu_time_base);
+        let mut out_video_enc = out_video_ctx.encoder().video().unwrap();
+        out_video_enc.set_format(ffmpeg_next::format::Pixel::YUV420P);
+        out_video_enc.set_aspect_ratio(aspect_ratio);
+        out_video_enc.set_width(u32::try_from(w).unwrap());
+        out_video_enc.set_height(u32::try_from(h).unwrap());
+        out_video_enc.set_time_base(emu_time_base);
+        let out_video_enc = out_video_enc.open().unwrap();
+        let out_vframe = FFVFrame::new(
+            out_video_enc.format(),
+            out_video_enc.width(),
+            out_video_enc.height(),
+        );
+        let (copy_format, is_native, stride) = match pixel_format {
+            retro_rs::libretro::retro_pixel_format::RETRO_PIXEL_FORMAT_0RGB1555 => {
+                (ffmpeg_next::format::Pixel::RGB555, true, 2)
+            }
+            retro_rs::libretro::retro_pixel_format::RETRO_PIXEL_FORMAT_XRGB8888 => {
+                (ffmpeg_next::format::Pixel::ZRGB, true, 4)
+            }
+            retro_rs::libretro::retro_pixel_format::RETRO_PIXEL_FORMAT_RGB565 => {
+                (ffmpeg_next::format::Pixel::RGB565, true, 2)
+            }
+            _other => (ffmpeg_next::format::Pixel::RGB24, false, 3),
+        };
+        let out_rgbframe = FFVFrame::new(
+            copy_format,
+            u32::try_from(w).unwrap(),
+            u32::try_from(h).unwrap(),
+        );
+
+        let converter = img_conv(
+            (u32::try_from(w).unwrap(), u32::try_from(h).unwrap()),
+            out_rgbframe.format(),
+            out_video_enc.format(),
+        )
+        .unwrap();
+        Self {
+            out_video_enc,
+            out_vframe,
+            out_rgbframe,
+            encoded_video,
+            converter,
+            emu_time_base,
+            native_pixel_format: is_native,
+            stride,
+        }
+    }
+    fn writeout(&mut self, output: &mut FFOut) {
+        let output_time_base = output.stream(0).unwrap().time_base();
+        while self
+            .out_video_enc
+            .receive_packet(&mut self.encoded_video)
+            .is_ok()
+        {
+            self.encoded_video.set_stream(0);
+            self.encoded_video
+                .rescale_ts(self.out_video_enc.time_base(), output_time_base);
+            self.encoded_video.write_interleaved(output).unwrap();
+        }
+    }
+    fn send_frame(&mut self, emu: &Emulator, frame_num: u64, output: &mut FFOut) {
+        // output one frame of video/audio, set_pts
+        // copy video to out_vframe
+        if self.native_pixel_format {
+            let pitch = emu.framebuffer_pitch();
+            let (w, h) = emu.framebuffer_size();
+            let stride = self.stride;
+            emu.peek_framebuffer(|fb| {
+                let data = self.out_rgbframe.data_mut(0);
+                for y in 0..h {
+                    data[(y * w * stride)..((y + 1) * w * stride)]
+                        .copy_from_slice(&fb[(y * pitch)..(y * pitch + w * stride)]);
+                }
+            })
+            .unwrap();
+        } else {
+            emu.copy_framebuffer_rgb888(self.out_rgbframe.data_mut(0))
+                .unwrap();
+        }
+        self.converter
+            .run(&self.out_rgbframe, &mut self.out_vframe)
+            .unwrap();
+        let frame_num = i64::try_from(frame_num).unwrap();
+        let frame_pts = frame_num.rescale(self.emu_time_base, self.out_video_enc.time_base());
+        self.out_vframe.set_pts(Some(frame_pts));
+        self.out_video_enc.send_frame(&self.out_vframe).unwrap();
+        self.writeout(output);
+    }
+    fn drain(&mut self, output: &mut FFOut) {
+        self.out_video_enc.send_eof().unwrap();
+        self.writeout(output);
     }
 }
-fn copy_video(
+
-    fb: &[u8],
+struct AudioState {
-    converter: &mut ffmpeg_next::software::scaling::Context,
+    out_audio_enc: ffmpeg_next::encoder::audio::Encoder,
-    rgbframe: &mut FFVFrame,
+    out_aframe: FFAFrame,
-    outframe: &mut FFVFrame,
+    in_aframe: FFAFrame,
-) {
+    encoded_audio: ffmpeg_next::Packet,
-    rgbframe.data_mut(0).copy_from_slice(fb);
+    audio_buf: ringbuf::LocalRb<ringbuf::storage::Heap<i16>>,
-    converter.run(rgbframe, outframe).unwrap();
+    audio_frame_out: i64,
+    audio_frame_in: i64,
+    resampler: ffmpeg_next::software::resampling::Context,
 }
 
+impl AudioState {
+    fn new(in_audio_sample_rate: i32, output: &mut FFOut) -> Self {
+        let out_audio_codec = ffmpeg_next::encoder::find(ffmpeg_next::codec::Id::AAC).unwrap();
+        let mut out_audio_ctx =
+            ffmpeg_next::codec::context::Context::new_with_codec(out_audio_codec);
+        // out_audio_ctx.debug(Debug::all());
+        let mut audio_params = ffmpeg_next::codec::Parameters::new();
+        unsafe {
+            let aps = audio_params.as_mut_ptr();
+            (*aps).codec_id = out_audio_codec.id().into();
+            (*aps).codec_type = ffmpeg_next::ffi::AVMediaType::AVMEDIA_TYPE_AUDIO;
+            (*aps).sample_rate = 48000;
+            (*aps).frame_size = 1024;
+            ffmpeg_next::ffi::av_channel_layout_default(&mut ((*aps).ch_layout), 2);
+        };
+        out_audio_ctx.set_parameters(audio_params).unwrap();
+        let _out_audio = output.add_stream_with(&out_audio_ctx).unwrap();
+        let encoded_audio = ffmpeg_next::Packet::empty();
+        let audio_time_base = Rational::new(1, 48000);
+        let mut out_audio_enc = out_audio_ctx.encoder().audio().unwrap();
+        out_audio_enc.set_channel_layout(ChannelLayout::default(2));
+        out_audio_enc.set_format(ffmpeg_next::format::Sample::F32(
+            ffmpeg_next::format::sample::Type::Planar,
+        ));
+        out_audio_enc.set_channel_layout(ffmpeg_next::ChannelLayout::STEREO);
+        out_audio_enc.set_time_base(audio_time_base);
+        out_audio_enc.set_rate(audio_time_base.1);
+        let out_audio_enc = out_audio_enc.open().unwrap();
+        let mut in_aframe = FFAFrame::new(
+            ffmpeg_next::format::Sample::I16(ffmpeg_next::format::sample::Type::Packed),
+            1024, // Just my choice of buffering rate
+            ffmpeg_next::ChannelLayout::STEREO,
+        );
+        in_aframe.set_rate(u32::try_from(in_audio_sample_rate).unwrap());
+        let mut out_aframe = FFAFrame::new(
+            out_audio_enc.format(),
+            out_audio_enc.frame_size() as usize,
+            out_audio_enc.channel_layout(),
+        );
+        out_aframe.set_rate(out_audio_enc.rate());
+        let resampler = in_aframe
+            .resampler(
+                out_aframe.format(),
+                out_aframe.channel_layout(),
+                out_aframe.rate(),
+            )
+            .unwrap();
+        let audio_buf = ringbuf::LocalRb::new(in_aframe.samples() * 2 * 20);
+
+        Self {
+            out_audio_enc,
+            out_aframe,
+            encoded_audio,
+            audio_buf,
+            audio_frame_out: 0,
+            audio_frame_in: 0,
+            resampler,
+            in_aframe,
+        }
+    }
+    fn writeout(&mut self, output: &mut FFOut) {
+        let output_time_base = output.stream(1).unwrap().time_base();
+        while self
+            .out_audio_enc
+            .receive_packet(&mut self.encoded_audio)
+            .is_ok()
+        {
+            self.encoded_audio.set_stream(1);
+            self.encoded_audio
+                .rescale_ts(self.out_audio_enc.time_base(), output_time_base);
+            self.encoded_audio.write_interleaved(output).unwrap();
+        }
+    }
+    fn resample_and_send(&mut self, output: &mut FFOut, drain: bool) {
+        match self.resampler.run(&self.in_aframe, &mut self.out_aframe) {
+            Ok(_) => {
+                self.in_aframe.set_pts(Some(self.audio_frame_in));
+                self.out_aframe.set_pts(Some(self.audio_frame_out));
+                self.audio_frame_in += i64::try_from(self.in_aframe.samples()).unwrap();
+                self.audio_frame_out += i64::try_from(self.out_aframe.samples()).unwrap();
+                self.out_audio_enc.send_frame(&self.out_aframe).unwrap();
+                self.writeout(output);
+            }
+            Err(e) => {
+                println!("Resampler error {e}");
+            }
+        }
+        loop {
+            let Some(delay) = self.resampler.delay() else {
+                break;
+            };
+            if delay.output < 524 && !drain {
+                break;
+            }
+            self.in_aframe.set_pts(Some(self.audio_frame_in));
+            self.out_aframe.set_pts(Some(self.audio_frame_out));
+            self.resampler.flush(&mut self.out_aframe).unwrap();
+            self.audio_frame_in += i64::try_from(self.in_aframe.samples()).unwrap();
+            self.audio_frame_out += i64::try_from(self.out_aframe.samples()).unwrap();
+            self.out_audio_enc.send_frame(&self.out_aframe).unwrap();
+            self.writeout(output);
+        }
+    }
+    fn send_frames(&mut self, emu: &Emulator, output: &mut FFOut) {
+        #[allow(unused_must_use)]
+        emu.peek_audio_sample(|samples| {
+            self.audio_buf.push_slice_overwrite(samples);
+            while self.audio_buf.occupied_len() >= self.in_aframe.samples() * 2 {
+                let (_, toconvert, _) = unsafe { self.in_aframe.data_mut(0).align_to_mut::<i16>() };
+                assert_eq!(self.audio_buf.pop_slice(toconvert), toconvert.len());
+                self.resample_and_send(output, false);
+            }
+        });
+    }
+    fn drain(&mut self, output: &mut FFOut) {
+        while self.audio_buf.occupied_len() > 0 {
+            let (_, toconvert, _) = unsafe { self.in_aframe.data_mut(0).align_to_mut::<i16>() };
+            let len = self.audio_buf.pop_slice(toconvert);
+            toconvert[len..].fill(0);
+            self.resample_and_send(output, true);
+        }
+        self.resample_and_send(output, true);
+        self.out_audio_enc.send_eof().unwrap();
+        self.writeout(output);
+    }
+}
+
+// bobl example: cargo run --bin genvideo examples/bobl.replay examples/bobl.mp4 cores/fceumm_libretro roms/bobl.nes
+// ff3 example: cargo run --bin genvideo examples/ff3v2.replay examples/ff3.mp4 cores/snes9x_libretro roms/ff3.nes
+
 fn main() {
     ffmpeg_next::init().unwrap();
-    ffmpeg_next::log::set_level(ffmpeg_next::log::Level::Trace);
+    ffmpeg_next::log::set_level(ffmpeg_next::log::Level::Warning);
     let args: Vec<_> = std::env::args().collect();
     let file =
-        std::fs::File::open(args.get(1).unwrap_or(&"examples/bobl.replay".to_string())).unwrap();
+        std::fs::File::open(args.get(1).unwrap_or(&"examples/ff3v2.replay".to_string())).unwrap();
-    let outfile = std::path::PathBuf::from(args.get(2).unwrap_or(&"examples/bobl.mp4".to_string()));
+    let outfile = std::path::PathBuf::from(args.get(2).unwrap_or(&"examples/ff3.mp4".to_string()));
     let corefile = args
         .get(3)
-        .unwrap_or(&"cores/fceumm_libretro".to_string())
+        .unwrap_or(&"cores/snes9x_libretro".to_string())
         .clone();
-    let romfile = args.get(4).unwrap_or(&"roms/bobl.nes".to_string()).clone();
+    let romfile = args.get(4).unwrap_or(&"roms/ff3.sfc".to_string()).clone();
     let mut emu = Emulator::create(Path::new(&corefile), Path::new(&romfile));
     let file = std::io::BufReader::new(file);
     let mut rply = decode(file).unwrap();
@@ -79,174 +336,45 @@ fn main() {
     }
     // run emu a tick to make sure we have right frame sizes, etc
     emu.run([retro_rs::Buttons::default(); 2]);
-
     let (w, h) = emu.framebuffer_size();
+    let pixel_format = emu.pixel_format();
+    assert!(emu.load(&rply.initial_state));
 
     let mut output = ffmpeg_next::format::output(&outfile).unwrap();
-    let emu_time_base =
+    let emu_video_framerate = emu.get_video_fps().to_i32().unwrap();
-        ffmpeg_next::util::rational::Rational::new(1, emu.get_video_fps().to_i32().unwrap());
+    let emu_time_base = Rational::new(1, emu_video_framerate);
-    let out_video_codec = ffmpeg_next::encoder::find(ffmpeg_next::codec::Id::H264).unwrap();
+    let audio_sample_rate = emu.get_audio_sample_rate().to_i32().unwrap();
-    let mut out_video_ctx = ffmpeg_next::codec::context::Context::new_with_codec(out_video_codec);
+    let aspect_ratio = Rational::from(f64::from(emu.get_aspect_ratio()));
-    // out_video_ctx.set_time_base(emu_time_base);
+    let mut video_state =
-    let mut video_params = ffmpeg_next::codec::Parameters::new();
+        VideoState::new(emu_time_base, aspect_ratio, w, h, pixel_format, &mut output);
-    unsafe {
+    let mut audio_state = AudioState::new(audio_sample_rate, &mut output);
-        let vps = video_params.as_mut_ptr();
-        (*vps).width = i32::try_from(w).unwrap();
-        (*vps).height = i32::try_from(h).unwrap();
-        (*vps).codec_id = out_video_codec.id().into();
-        (*vps).codec_type = ffmpeg_next::ffi::AVMediaType::AVMEDIA_TYPE_VIDEO;
-    };
-    out_video_ctx.set_parameters(video_params).unwrap();
-    let _out_video = output.add_stream_with(&out_video_ctx).unwrap();
-    let mut encoded_video = ffmpeg_next::Packet::empty();
-    // out_video.set_time_base(emu_time_base);
-    let mut out_video_enc = out_video_ctx.encoder().video().unwrap();
-    out_video_enc.set_format(ffmpeg_next::format::Pixel::YUV420P);
-    out_video_enc.set_width(u32::try_from(w).unwrap());
-    out_video_enc.set_height(u32::try_from(h).unwrap());
-    out_video_enc.set_time_base(emu_time_base);
-    let mut out_video_enc = out_video_enc.open().unwrap();
-    let out_audio_codec = ffmpeg_next::encoder::find(ffmpeg_next::codec::Id::AAC).unwrap();
-    let mut out_audio_ctx = ffmpeg_next::codec::context::Context::new_with_codec(out_audio_codec);
-    // out_audio_ctx.debug(Debug::all());
-    let mut audio_params = ffmpeg_next::codec::Parameters::new();
-    unsafe {
-        let aps = audio_params.as_mut_ptr();
-        (*aps).codec_id = out_audio_codec.id().into();
-        (*aps).codec_type = ffmpeg_next::ffi::AVMediaType::AVMEDIA_TYPE_AUDIO;
-        (*aps).frame_size = (emu.get_audio_sample_rate() / emu.get_video_fps().floor())
-            .to_i32()
-            .unwrap();
-        (*aps).sample_rate = emu.get_audio_sample_rate().to_i32().unwrap();
-        (*aps).channels = 2;
-    };
-    out_audio_ctx.set_parameters(audio_params).unwrap();
-    // out_audio_ctx.set_time_base(ffmpeg_next::util::rational::Rational::new(
-    //     1,
-    //     emu.get_audio_sample_rate() as i32,
-    // ));
-    let _out_audio = output.add_stream_with(&out_audio_ctx).unwrap();
-    let mut encoded_audio = ffmpeg_next::Packet::empty();
-    // out_audio.set_time_base(ffmpeg_next::util::rational::Rational::new(
-    //     1,
-    //     emu.get_audio_sample_rate() as i32,
-    // ));
-    let audio_time_base = ffmpeg_next::util::rational::Rational::new(
-        1,
-        emu.get_audio_sample_rate().to_i32().unwrap(),
-    );
-    let mut out_audio_enc = out_audio_ctx.encoder().audio().unwrap();
-    out_audio_enc.set_channels(2);
-    out_audio_enc.set_format(ffmpeg_next::format::Sample::F32(
-        ffmpeg_next::format::sample::Type::Planar,
-    ));
-    out_audio_enc.set_channel_layout(ffmpeg_next::ChannelLayout::STEREO);
-    out_audio_enc.set_time_base(audio_time_base);
-    out_audio_enc.set_rate(emu.get_audio_sample_rate().to_i32().unwrap());
-    let mut out_audio_enc = out_audio_enc.open().unwrap();
-    let mut out_vframe = FFVFrame::new(
-        out_video_enc.format(),
-        out_video_enc.width(),
-        out_video_enc.height(),
-    );
-    let mut out_rgbframe = FFVFrame::new(
-        ffmpeg_next::format::Pixel::RGB24,
-        u32::try_from(w).unwrap(),
-        u32::try_from(h).unwrap(),
-    );
-    let mut out_aframe = FFAFrame::new(
-        out_audio_enc.format(),
-        out_audio_enc.frame_size() as usize,
-        out_audio_enc.channel_layout(),
-    );
     output.write_header().unwrap();
-    assert!(emu.load(&rply.initial_state));
+    // video_state
-    let video_stream_time_base = output.stream(0).unwrap().time_base();
+    //     .encoded_video
-    let audio_stream_time_base = output.stream(1).unwrap().time_base();
+    //     .set_time_base(video_stream_time_base);
-    encoded_video.set_time_base(video_stream_time_base);
+    // audio_state
-    encoded_audio.set_time_base(audio_stream_time_base);
+    //     .encoded_audio
+    //     .set_time_base(audio_stream_time_base);
 
     let mut frame = Frame::default();
-    let mut fb = vec![0_u8; w * h * 3];
-    let mut converter = img_conv(
-        (u32::try_from(w).unwrap(), u32::try_from(h).unwrap()),
-        out_rgbframe.format(),
-        out_video_enc.format(),
-    )
-    .unwrap();
-    let mut audio_buf = ringbuf::LocalRb::new(out_aframe.samples() * 2 * 20);
-    let mut frame_audio_buf = vec![0_i16; out_aframe.samples() * 2];
-    let mut audio_frame = 0;
     while let Ok(()) = rply
         .read_frame(&mut frame)
         .inspect_err(|e| println!("Err: {e}"))
     {
-        use ffmpeg_next::util::mathematics::Rescale;
         let buttons = frame_to_buttons(&frame);
         emu.run(buttons);
-        emu.copy_framebuffer_rgb888(&mut fb).unwrap();
+        video_state.send_frame(&emu, rply.frame_number, &mut output);
+        audio_state.send_frames(&emu, &mut output);
         if !frame.checkpoint_bytes.is_empty() {
             assert!(emu.load(&frame.checkpoint_bytes));
         }
-        // output one frame of video/audio, set_pts
+
-        // copy video to out_vframe
-        copy_video(&fb, &mut converter, &mut out_rgbframe, &mut out_vframe);
-        let frame_num = i64::try_from(rply.frame_number).unwrap();
-        let frame_pts = frame_num.rescale(emu_time_base, out_video_enc.time_base());
-        out_vframe.set_pts(Some(frame_pts));
-        out_video_enc.send_frame(&out_vframe).unwrap();
-        // copy audio to out_aframe, set_pts
-        // maybe in a loop?
-        #[allow(unused_must_use)]
-        emu.peek_audio_sample(|samples| {
-            audio_buf.push_slice_overwrite(samples);
-            while audio_buf.occupied_len() >= out_aframe.samples() * 2 {
-                assert_eq!(
-                    audio_buf.pop_slice(&mut frame_audio_buf),
-                    frame_audio_buf.len()
-                );
-                copy_audio(&frame_audio_buf, &mut out_aframe);
-                out_aframe.set_pts(Some(audio_frame));
-                audio_frame += i64::try_from(out_aframe.samples()).unwrap();
-                out_audio_enc.send_frame(&out_aframe).unwrap();
-            }
-        });
-        while out_video_enc.receive_packet(&mut encoded_video).is_ok() {
-            encoded_video.set_stream(0);
-            encoded_video.rescale_ts(out_video_enc.time_base(), video_stream_time_base);
-            encoded_video.write_interleaved(&mut output).unwrap();
-        }
-        while out_audio_enc.receive_packet(&mut encoded_audio).is_ok() {
-            encoded_audio.set_stream(1);
-            encoded_audio.rescale_ts(out_audio_enc.time_base(), audio_stream_time_base);
-            encoded_audio.write_interleaved(&mut output).unwrap();
-        }
         if Some(rply.frame_number) == rply.header.frame_count() {
             break;
         }
     }
-    while audio_buf.occupied_len() >= out_aframe.samples() {
+    audio_state.drain(&mut output);
-        let len = audio_buf.pop_slice(&mut frame_audio_buf);
+    video_state.drain(&mut output);
-        frame_audio_buf[len..].fill(0);
-        out_aframe.set_pts(Some(audio_frame));
-        audio_frame += i64::try_from(len / 2).unwrap();
-        copy_audio(&frame_audio_buf, &mut out_aframe);
-        out_audio_enc.send_frame(&out_aframe).unwrap();
-    }
-
-    out_video_enc.send_eof().unwrap();
-    out_audio_enc.send_eof().unwrap();
-
-    while out_video_enc.receive_packet(&mut encoded_video).is_ok() {
-        encoded_video.set_stream(0);
-        encoded_video.rescale_ts(out_video_enc.time_base(), video_stream_time_base);
-        encoded_video.write_interleaved(&mut output).unwrap();
-    }
-    while out_audio_enc.receive_packet(&mut encoded_audio).is_ok() {
-        encoded_audio.set_stream(1);
-        encoded_audio.rescale_ts(out_audio_enc.time_base(), audio_stream_time_base);
-        encoded_audio.write_interleaved(&mut output).unwrap();
-    }
     output.write_trailer().unwrap();
 }
 

upgradev0/src/main.rs

@@ -7,14 +7,14 @@ use std::rc::Rc;
 fn main() {
     let args: Vec<_> = std::env::args().collect();
     let file =
-        std::fs::File::open(args.get(1).unwrap_or(&"examples/v0.replay".to_string())).unwrap();
+        std::fs::File::open(args.get(1).unwrap_or(&"examples/ff3.replay".to_string())).unwrap();
     let outfile =
-        std::fs::File::create(args.get(2).unwrap_or(&"examples/v2.replay".to_string())).unwrap();
+        std::fs::File::create(args.get(2).unwrap_or(&"examples/ff3v2.replay".to_string())).unwrap();
     let corefile = args
         .get(3)
-        .unwrap_or(&"cores/fceumm_libretro".to_string())
+        .unwrap_or(&"cores/snes9x_libretro".to_string())
         .clone();
-    let romfile = args.get(4).unwrap_or(&"roms/demo.nes".to_string()).clone();
+    let romfile = args.get(4).unwrap_or(&"roms/ff3.sfc".to_string()).clone();
     let mut emu = Emulator::create(Path::new(&corefile), Path::new(&romfile));
     let file = std::io::BufReader::new(file);
     let mut outfile = std::io::BufWriter::new(outfile);