From f5bc322862a6b74a455dc6318925e1d4f99762be Mon Sep 17 00:00:00 2001 From: Andy Killorin <37423245+Speedy6451@users.noreply.github.com> Date: Thu, 2 Jan 2025 19:42:05 -0500 Subject: [PATCH] switched to jpeg from h264 compression 10x larger over the wire but no dependency on libstdc++ --- decoder/Cargo.lock | 16 ++++++++++++++++ decoder/Cargo.toml | 1 + decoder/src/lib.rs | 43 +++++++++++-------------------------------- encoder/Cargo.lock | 1 + encoder/Cargo.toml | 1 + encoder/src/main.rs | 40 ++++++++++++++++------------------------ 6 files changed, 46 insertions(+), 56 deletions(-) diff --git a/decoder/Cargo.lock b/decoder/Cargo.lock index 2e5c965..4e21f68 100644 --- a/decoder/Cargo.lock +++ b/decoder/Cargo.lock @@ -79,6 +79,7 @@ version = "0.1.0" dependencies = [ "openh264", "tokio", + "zune-jpeg", ] [[package]] @@ -453,3 +454,18 @@ name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zune-core" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f423a2c17029964870cfaabb1f13dfab7d092a62a29a89264f4d36990ca414a" + +[[package]] +name = "zune-jpeg" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99a5bab8d7dedf81405c4bb1f2b83ea057643d9cb28778cea9eecddeedd2e028" +dependencies = [ + "zune-core", +] diff --git a/decoder/Cargo.toml b/decoder/Cargo.toml index d22c7d7..b3f1403 100644 --- a/decoder/Cargo.toml +++ b/decoder/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" [dependencies] openh264 = "0.6.6" tokio = { version = "1.42.0", features = ["full"] } +zune-jpeg = "0.4.14" [lib] crate-type = ["cdylib","lib"] diff --git a/decoder/src/lib.rs b/decoder/src/lib.rs index ad0f27c..021ee8e 100644 --- a/decoder/src/lib.rs +++ b/decoder/src/lib.rs @@ -1,51 +1,30 @@ use std::{mem::transmute, net::{self, SocketAddr}, slice, sync::{Mutex, OnceLock}}; use openh264::{decoder::Decoder, formats::YUVSource}; +use zune_jpeg::{zune_core::{colorspace::ColorSpace, options::DecoderOptions}, JpegDecoder}; #[no_mangle] pub extern fn add(a: u64, b: u64) -> u64 { a + b } -static DECODER: OnceLock> = OnceLock::new(); - #[no_mangle] -/// reset h264 decoder -/// -/// call before opening new stream -pub extern fn reset_decoder() -> u64 { - // calls constructor twice in the reset case, not big deal - if let Err(_) = DECODER.set(Mutex::new(Decoder::new().unwrap())) { - *DECODER.get().unwrap().lock().unwrap() = Decoder::new().unwrap(); - } - - 0 -} - -#[no_mangle] -/// decode h264 packet of given length +/// decode jpeg image of given length /// /// image format is 0RGB -/// -/// returns 1 if the packet contained a new frame, otherwise 0 -pub extern fn decode_h264(image: &mut u32, packet: &u8, length: u32) -> u64 { - let decoder = DECODER.get_or_init(|| Mutex::new(Decoder::new().unwrap())); - let mut decoder = decoder.lock().unwrap(); - +pub extern fn decode(image: &mut u32, packet: &u8, length: u32) -> u64 { let packet = unsafe {slice::from_raw_parts(packet, length as usize)}; let image: &mut [u8; 4*320*240] = unsafe{transmute(image)}; - if let Ok(Some(frame)) = decoder.decode(packet) { - let mut buf = [0u8; 3*320*240]; - frame.write_rgb8(&mut buf); - // 0RGB conversion, avoids need to modify openh264 - for (buffer, image) in buf.chunks_exact_mut(3).zip(image.chunks_exact_mut(4)) { - image[1] = buffer[0]; - image[2] = buffer[1]; - image[3] = buffer[2]; - } + let options = DecoderOptions::new_fast().jpeg_set_out_colorspace(ColorSpace::RGB); + let mut frame = JpegDecoder::new_with_options(packet, options).decode().unwrap(); - return 1; + // 0RGB conversion, easier than shifting and zeroing A + for (buffer, image) in frame.chunks_exact_mut(3).zip(image.chunks_exact_mut(4)) { + image[0] = 0; + image[1] = buffer[0]; + image[2] = buffer[1]; + image[3] = buffer[2]; } 0 diff --git a/encoder/Cargo.lock b/encoder/Cargo.lock index e654b64..99582bc 100644 --- a/encoder/Cargo.lock +++ b/encoder/Cargo.lock @@ -423,6 +423,7 @@ dependencies = [ "nokhwa", "openh264", "tokio", + "zune-jpeg", ] [[package]] diff --git a/encoder/Cargo.toml b/encoder/Cargo.toml index 65c9864..abf915e 100644 --- a/encoder/Cargo.toml +++ b/encoder/Cargo.toml @@ -9,3 +9,4 @@ image = "0.25.5" nokhwa = { version = "0.10.7", features = ["input-native"] } openh264 = "0.6.6" tokio = { version = "1.42.0", features = ["full"] } +zune-jpeg = "0.4.14" diff --git a/encoder/src/main.rs b/encoder/src/main.rs index 15f2247..a1f1343 100644 --- a/encoder/src/main.rs +++ b/encoder/src/main.rs @@ -1,14 +1,13 @@ -use std::{net::SocketAddr, result, sync::Arc, thread::{self, sleep}, time::Duration}; +use std::{array::from_ref, net::SocketAddr, result, sync::Arc, thread::{self, sleep}, time::Duration}; -use image::{ImageBuffer, Rgb}; +use image::{codecs::jpeg::JpegEncoder, ImageBuffer, Rgb}; use nokhwa::{pixel_format::RgbFormat, utils::{ApiBackend, RequestedFormat, RequestedFormatType, Resolution}, Camera}; use anyhow::{Context, Ok, Result}; -use openh264::{encoder::{Encoder, EncoderConfig, UsageType}, formats::{RgbSliceU8, YUVBuffer}, nal_units, OpenH264API}; use tokio::{io::AsyncWriteExt, net::{TcpListener, TcpStream}, runtime::Runtime, sync::{Notify, RwLock}, task::LocalSet}; fn main() -> Result<()>{ let await_frame = Arc::new(Notify::new()); - let latest_frame = Arc::new(RwLock::new(YUVBuffer::new(0, 0))); + let latest_frame = Arc::new(RwLock::new(Vec::new())); { let runtime = Runtime::new()?; @@ -34,7 +33,7 @@ fn main() -> Result<()>{ loop {} } -async fn camera_manager(await_frame: Arc, latest_frame: Arc>) -> Result<()>{ +async fn camera_manager(await_frame: Arc, latest_frame: Arc>>) -> Result<()>{ let cameras = nokhwa::query(ApiBackend::Auto)?; let camera = cameras.get(0).context("no cameras")?; @@ -47,41 +46,34 @@ async fn camera_manager(await_frame: Arc, latest_frame: Arc, Vec> = frame.decode_image::()?; - let buf = RgbSliceU8::new(frame.as_raw(), (resolution.width_x as usize, resolution.height_y as usize)); - let yuv = YUVBuffer::from_rgb8_source(buf); - *latest_frame.write().await = yuv; + let mut output = Vec::new(); + + let mut encoder = JpegEncoder::new_with_quality(&mut output, 30); + + encoder.encode_image(&frame).unwrap(); + + *latest_frame.write().await = output; await_frame.notify_waiters(); } } -async fn stream_video(await_frame: Arc, latest_frame: Arc>, mut client: TcpStream) -> Result<()>{ - - let mut encoder = Encoder::with_api_config( - OpenH264API::from_source(), - EncoderConfig::new() - .usage_type(UsageType::CameraVideoRealTime - ))?; - - +async fn stream_video(await_frame: Arc, latest_frame: Arc>>, mut client: TcpStream) -> Result<()>{ loop { await_frame.notified().await; - let data = encoder.encode(&*latest_frame.read().await)?; - let data = data.to_vec(); + let data = latest_frame.read().await; println!("len: {}", data.len()); let len = data.len(); - let data2 = data.clone(); - drop(data); - client.write(&data2).await?; + // holding data across await, likely nonissue client.write_u32(len as u32).await?; + client.write(&data).await?; } } -async fn server(await_frame: Arc, latest_frame: Arc>) -> Result<()> { +async fn server(await_frame: Arc, latest_frame: Arc>>) -> Result<()> { let port = 2993; let listener = TcpListener::bind(format!("0.0.0.0:{port}")).await?; println!("listening on {port}");