guanyuankai
/
bonus-edge-proxy
Template
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修复dma问题

This commit is contained in:
zengqile 2026-01-26 18:32:52 +08:00
parent 005168b8e1
commit ea5e23da86
11 changed files with 1070 additions and 544 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
CMakeLists.txt
View File

@ -13,10 +13,16 @@ find_package(PahoMqttCpp REQUIRED)
find_package(SQLite3 REQUIRED) find_package(SQLite3 REQUIRED)
find_package(PkgConfig REQUIRED) find_package(PkgConfig REQUIRED)
# pkg_check_modules(GST REQUIRED gstreamer-1.0 gstreamer-app-1.0 gstreamer-allocators-1.0)
pkg_check_modules(GST REQUIRED gstreamer-1.0 gstreamer-app-1.0 gstreamer-allocators-1.0 gstreamer-video-1.0) pkg_check_modules(GST REQUIRED gstreamer-1.0 gstreamer-app-1.0 gstreamer-allocators-1.0 gstreamer-video-1.0)
find_package(OpenCV REQUIRED) find_package(OpenCV REQUIRED)
# 查找 FFmpeg 库
find_package(PkgConfig REQUIRED)
pkg_check_modules(AVCODEC REQUIRED libavcodec)
pkg_check_modules(AVFORMAT REQUIRED libavformat)
pkg_check_modules(AVUTIL REQUIRED libavutil)
pkg_check_modules(SWSCALE REQUIRED libswscale)
add_subdirectory(src/vendor/crow) add_subdirectory(src/vendor/crow)
set(BYTETRACK_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/algorithm/bytetrack/src) set(BYTETRACK_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/algorithm/bytetrack/src)
@ -82,6 +88,7 @@ add_library(edge_proxy_lib STATIC
src/rknn/rkYolov8.cc src/rknn/rkYolov8.cc
src/rknn/preprocess.cc src/rknn/preprocess.cc
src/rknn/postprocess.cc src/rknn/postprocess.cc
src/rknn/ffmpeg_rga_decoder.cpp
src/videoServiceManager/video_service_manager.cc src/videoServiceManager/video_service_manager.cc
src/algorithm/IntrusionModule.cc src/algorithm/IntrusionModule.cc
src/algorithm/HumanDetectionModule.cc src/algorithm/HumanDetectionModule.cc
@ -96,7 +103,13 @@ target_include_directories(edge_proxy_lib PUBLIC
/usr/include/rga /usr/include/rga
${EIGEN3_INCLUDE_DIRS} # Eigen3 头文件 ${EIGEN3_INCLUDE_DIRS} # Eigen3 头文件
${CMAKE_CURRENT_SOURCE_DIR}/src/algorithm/bytetrack/include # ByteTrack 头文件 ${CMAKE_CURRENT_SOURCE_DIR}/src/algorithm/bytetrack/include
#ffmpeg
${AVCODEC_INCLUDE_DIRS}
${AVFORMAT_INCLUDE_DIRS}
${AVUTIL_INCLUDE_DIRS}
${SWSCALE_INCLUDE_DIRS}
) )
target_link_libraries(edge_proxy_lib PRIVATE target_link_libraries(edge_proxy_lib PRIVATE
@ -113,6 +126,12 @@ target_link_libraries(edge_proxy_lib PRIVATE
rga rga
${OpenCV_LIBS} ${OpenCV_LIBS}
${GST_LIBRARIES} ${GST_LIBRARIES}
#ffmpeg
${AVCODEC_LIBRARIES}
${AVFORMAT_LIBRARIES}
${AVUTIL_LIBRARIES}
${SWSCALE_LIBRARIES}
) )

434
config/video_config.json
View File

@ -1,213 +1,225 @@
{ {
"video_service": { "video_service": {
"enabled": true "enabled": true
}, },
"video_streams": [ "video_streams": [
{ {
"enabled": true, "enabled": true,
"id": "cam_01_intrusion", "id": "cam_01_intrusion",
"input_url": "rtsp://admin:hzx12345@192.168.1.10:554/Streaming/Channels/3101", "input_url": "rtsp://admin:hzx12345@192.168.1.10:554/Streaming/Channels/3101",
"module_config": { "module_config": {
"class_num": 3, "class_num": 3,
"intrusion_zone": [ "intrusion_zone": [
[ [
4, 4,
8 8
], ],
[ [
1057, 1057,
13 13
], ],
[ [
1053, 1053,
267 267
], ],
[ [
1347, 1347,
289 289
], ],
[ [
1365, 1365,
8 8
], ],
[ [
1911, 1911,
6 6
], ],
[ [
1898, 1898,
1077 1077
], ],
[ [
7, 7,
1077 1077
] ]
], ],
"label_path": "/app/models/human.txt", "label_path": "/app/models/human.txt",
"light_device_ids": [ "light_device_ids": [
"351", "351",
"352", "352",
"353", "353",
"349", "349",
"354", "354",
"350" "350"
], ],
"mode": 1, "mode": 1,
"model_path": "/app/models/human_detection.rknn", "model_path": "/app/models/human_detection.rknn",
"rknn_thread_num": 3, "rknn_thread_num": 3,
"time_threshold_sec": 3 "time_threshold_sec": 3
}, },
"module_type": "human_detection", "module_type": "human_detection",
"output_rtsp": "rtsp://127.0.0.1:8554/ch1901" "output_rtsp": "rtsp://127.0.0.1:8554/ch1901"
}, },
{ {
"enabled": true, "enabled": true,
"id": "cam_02_intrusion", "id": "cam_02_intrusion",
"input_url": "rtsp://admin:hzx12345@192.168.1.10:554/Streaming/Channels/3201", "input_url": "rtsp://admin:hzx12345@192.168.1.10:554/Streaming/Channels/3201",
"module_config": { "module_config": {
"class_num": 3, "class_num": 3,
"intrusion_zone": [ "intrusion_zone": [
[ [
11, 11,
13 13
], ],
[ [
1907, 1907,
13 13
], ],
[ [
1911, 1911,
1074 1074
], ],
[ [
7, 7,
1070 1070
] ]
], ],
"label_path": "/app/models/human.txt", "label_path": "/app/models/human.txt",
"light_device_ids": [ "light_device_ids": [
"351", "351",
"352", "352",
"353", "353",
"349", "349",
"354", "354",
"350" "350"
], ],
"mode": 1, "mode": 1,
"model_path": "/app/models/human_detection.rknn", "model_path": "/app/models/human_detection.rknn",
"rknn_thread_num": 3, "rknn_thread_num": 3,
"time_threshold_sec": 3 "time_threshold_sec": 3
}, },
"module_type": "human_detection", "module_type": "human_detection",
"output_rtsp": "rtsp://127.0.0.1:8554/ch1801" "output_rtsp": "rtsp://127.0.0.1:8554/ch1801"
}, },
{ {
"enabled": true, "enabled": true,
"id": "cam_03_intrusion", "id": "cam_03_intrusion",
"input_url": "rtsp://admin:hzx12345@192.168.1.10:554/Streaming/Channels/1601", "input_url": "rtsp://admin:hzx12345@192.168.1.10:554/Streaming/Channels/1601",
"module_config": { "module_config": {
"class_num": 3, "class_num": 3,
"intrusion_zone": [ "intrusion_zone": [
[ [
20, 20,
19 19
], ],
[ [
1891, 1891,
35 35
], ],
[ [
1433, 1433,
466 466
], ],
[ [
1006, 1006,
619 619
], ],
[ [
1210, 1210,
1001 1001
], ],
[ [
1201, 1201,
1072 1072
], ],
[ [
2, 2,
1077 1077
] ]
], ],
"label_path": "/app/models/human.txt", "label_path": "/app/models/human.txt",
"model_path": "/app/models/human_detection.rknn", "model_path": "/app/models/human_detection.rknn",
"rknn_thread_num": 3, "rknn_thread_num": 3,
"time_threshold_sec": 3 "time_threshold_sec": 3
}, },
"module_type": "human_detection", "module_type": "human_detection",
"output_rtsp": "rtsp://127.0.0.1:8554/ch2201" "output_rtsp": "rtsp://127.0.0.1:8554/ch2201"
}, },
{ {
"enabled": false, "enabled": true,
"id": "cam_04_intrusion", "id": "cam_05_intrusion",
"input_url": "rtsp://admin:hzx12345@192.168.1.10:554/Streaming/Channels/1101", "input_url": "rtsp://admin:hzx12345@192.168.1.10:554/Streaming/Channels/1501",
"module_config": { "module_config": {
"class_num": 3, "class_num": 3,
"intrusion_zone": [ "intrusion_zone": [
[ [
400, 20,
400 19
], ],
[ [
800, 1891,
400 35
], ],
[ [
900, 1433,
600 466
], ],
[ [
300, 1006,
600 619
] ],
], [
"label_path": "/app/models/human.txt", 1210,
"model_path": "/app/models/human_detection.rknn", 1001
"rknn_thread_num": 3, ],
"time_threshold_sec": 3 [
}, 1201,
"module_type": "human_detection", 1072
"output_rtsp": "rtsp://127.0.0.1:8554/ch1201" ],
}, [
{ 2,
"enabled": false, 1077
"id": "cam_01_intrusion", ]
"input_url": "rtsp://admin:hzx12345@192.168.1.10:554/Streaming/Channels/1101", ],
"module_config": { "label_path": "/app/models/human.txt",
"class_num": 3, "model_path": "/app/models/human_detection.rknn",
"intrusion_zone": [ "rknn_thread_num": 3,
[ "time_threshold_sec": 3
400, },
400 "module_type": "human_detection",
], "output_rtsp": "rtsp://127.0.0.1:8554/ch1501"
[ },
800, {
400 "enabled": false,
], "id": "cam_06_intrusion",
[ "input_url": "rtsp://admin:hzx12345@192.168.1.10:554/Streaming/Channels/1101",
900, "module_config": {
600 "class_num": 3,
], "intrusion_zone": [
[ [
300, 400,
600 400
] ],
], [
"label_path": "/app/models/human.txt", 800,
"model_path": "/app/models/human_detection.rknn", 400
"rknn_thread_num": 3, ],
"time_threshold_sec": 3 [
}, 900,
"module_type": "human_detection", 600
"output_rtsp": "rtsp://127.0.0.1:8554/ch1101" ],
} [
] 300,
600
]
],
"label_path": "/app/models/human.txt",
"model_path": "/app/models/human_detection.rknn",
"rknn_thread_num": 3,
"time_threshold_sec": 3
},
"module_type": "human_detection",
"output_rtsp": "rtsp://127.0.0.1:8554/ch1101"
}
]
} }

BIN
cs.mp4 Normal file
View File

Binary file not shown.

138
src/rknn/dma_allocator.hpp
View File

@ -1,115 +1,109 @@
#ifndef DMA_ALLOCATOR_HPP #pragma once
#define DMA_ALLOCATOR_HPP
#include <fcntl.h> #include <fcntl.h>
#include <linux/types.h> #include <linux/dma-heap.h>
#include <stdio.h>
#include <string.h>
#include <sys/ioctl.h> #include <sys/ioctl.h>
#include <sys/mman.h> #include <sys/mman.h>
#include <unistd.h> #include <unistd.h>
#include <cstring>
#include <iostream> #include <iostream>
#include <opencv2/opencv.hpp> #include <string>
#include "im2d.h" #include "im2d.h"
#include "rga.h" #include "rga.h"
#define DMA_HEAP_IOCTL_MAGIC 'H'
struct dma_heap_allocation_data {
__u64 len;
__u32 fd;
__u32 fd_flags;
__u64 heap_flags;
};
#define DMA_HEAP_IOCTL_ALLOC _IOWR(DMA_HEAP_IOCTL_MAGIC, 0, struct dma_heap_allocation_data)
class DmaBuffer { class DmaBuffer {
public: public:
void* vaddr = nullptr; int fd;
int fd = -1; void* vaddr;
rga_buffer_handle_t handle = 0; size_t size;
size_t size = 0; size_t actual_alloc_size;
int width = 0;
int height = 0;
int format = 0;
DmaBuffer(int w, int h, int fmt) : width(w), height(h), format(fmt) { int width;
int bpp = (fmt == RK_FORMAT_RGB_888 || fmt == RK_FORMAT_BGR_888) ? 3 : 4; int height;
// 4K 对齐 // 构造函数 1: 仅大小
size_t raw_size = w * h * bpp; DmaBuffer(size_t size, const std::string& heap_name = "/dev/dma_heap/system-uncached") {
size = (raw_size + 4095) & (~4095); this->width = 0;
this->height = 0;
allocate(size, heap_name);
}
if (alloc_dma_buffer(size, &fd, &vaddr) < 0) { // 构造函数 2: 宽高
std::cerr << "[DmaBuffer] Error: Failed to allocate DMA buffer!" << std::endl; DmaBuffer(int width, int height, int format,
return; const std::string& heap_name = "/dev/dma_heap/system-uncached") {
this->width = width;
this->height = height;
size_t alloc_size = 0;
if (format == RK_FORMAT_RGB_888 || format == RK_FORMAT_BGR_888) {
alloc_size = width * height * 3;
} else if (format == RK_FORMAT_YCbCr_420_SP || format == RK_FORMAT_YCrCb_420_SP) {
alloc_size = width * height * 3 / 2;
} else {
alloc_size = width * height * 4;
} }
handle = (rga_buffer_handle_t)fd; allocate(alloc_size, heap_name);
} }
~DmaBuffer() { ~DmaBuffer() {
if (vaddr && vaddr != MAP_FAILED) if (vaddr != MAP_FAILED && vaddr != nullptr)
munmap(vaddr, size); munmap(vaddr, actual_alloc_size);
if (fd >= 0) if (fd >= 0)
close(fd); close(fd);
} }
rga_buffer_t getRgaBuffer() { bool isValid() const {
return wrapbuffer_fd(fd, width, height, format); return fd >= 0;
} }
bool isValid() const { rga_buffer_t getRgaBuffer() {
return fd >= 0 && vaddr != nullptr; return wrapbuffer_fd(fd, width, height, RK_FORMAT_RGB_888, width, height);
} }
private: private:
int alloc_dma_buffer(size_t size, int* fd, void** vaddr) { void allocate(size_t req_size, const std::string& heap_name) {
// [关键修复] 调整优先级 // [关键] 向上对齐到 4KB 页大小
// 1. system-uncached-dma32: 物理地址 < 4G (解决RGA报错)，且非Root用户可访问 long page_size = sysconf(_SC_PAGESIZE);
// 2. cma: 物理地址 < 4G，但通常需要 Root 权限 if (page_size < 0)
// 3. system-uncached: 最后的退路，但可能会分配到 > 4G 导致 RGA 崩溃 page_size = 4096;
const char* heap_paths[] = {"/dev/dma_heap/system-uncached-dma32", // 首选！
"/dev/dma_heap/system-dma32",
"/dev/dma_heap/cma", // 需要 Root
"/dev/dma_heap/linux,cma", "/dev/dma_heap/system-uncached"};
int heap_fd = -1; this->actual_alloc_size = (req_size + page_size - 1) & ~(page_size - 1);
for (const char* path : heap_paths) { this->size = req_size;
heap_fd = open(path, O_RDWR | O_CLOEXEC);
if (heap_fd >= 0) { this->vaddr = MAP_FAILED;
// std::cout << "[DmaBuffer] Success: Using heap " << path << std::endl; this->fd = -1;
break;
int heap_fd = open(heap_name.c_str(), O_RDONLY | O_CLOEXEC);
if (heap_fd < 0) {
if (heap_name.find("system") != std::string::npos) {
heap_fd = open("/dev/dma_heap/cma", O_RDONLY | O_CLOEXEC);
}
if (heap_fd < 0) {
perror("Failed to open dma heap");
return;
} }
} }
if (heap_fd < 0) {
std::cerr << "[DmaBuffer] Fatal: Could not open any suitable DMA heap! (Check "
"permissions for /dev/dma_heap/)"
<< std::endl;
return -1;
}
struct dma_heap_allocation_data data = {0}; struct dma_heap_allocation_data data = {0};
data.len = size; data.len = this->actual_alloc_size;
data.fd_flags = O_CLOEXEC | O_RDWR; data.fd_flags = O_RDWR | O_CLOEXEC;
if (ioctl(heap_fd, DMA_HEAP_IOCTL_ALLOC, &data) < 0) { if (ioctl(heap_fd, DMA_HEAP_IOCTL_ALLOC, &data) < 0) {
std::cerr << "[DmaBuffer] Error: DMA allocation ioctl failed." << std::endl; perror("dma heap alloc failed");
close(heap_fd); close(heap_fd);
return -1; return;
} }
*fd = data.fd; this->fd = data.fd;
close(heap_fd); close(heap_fd);
*vaddr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, *fd, 0); this->vaddr =
if (*vaddr == MAP_FAILED) { mmap(NULL, this->actual_alloc_size, PROT_READ | PROT_WRITE, MAP_SHARED, this->fd, 0);
close(*fd); if (this->vaddr == MAP_FAILED) {
*fd = -1; perror("mmap failed");
return -1;
} }
return 0;
} }
}; };
#endif

355
src/rknn/ffmpeg_rga_decoder.cpp Normal file
View File

@ -0,0 +1,355 @@
#include "ffmpeg_rga_decoder.h"
#include <opencv2/imgproc.hpp>
#include <opencv2/videoio.hpp>
#include "spdlog/spdlog.h"
// 结构体定义
struct RgaSrcInfo {
int fd;
int wstride;
int hstride;
void* vaddr;
int format;
};
FFmpegRGADecoder::FFmpegRGADecoder() {
pkt_ = av_packet_alloc();
frame_ = av_frame_alloc();
}
FFmpegRGADecoder::~FFmpegRGADecoder() {
release();
if (pkt_)
av_packet_free(&pkt_);
if (frame_)
av_frame_free(&frame_);
}
void FFmpegRGADecoder::cleanUp() {
if (dec_ctx_) {
avcodec_free_context(&dec_ctx_);
dec_ctx_ = nullptr;
}
if (fmt_ctx_) {
avformat_close_input(&fmt_ctx_);
fmt_ctx_ = nullptr;
}
is_opened_ = false;
}
void FFmpegRGADecoder::release() {
cleanUp();
output_dma_buf_.reset();
}
bool FFmpegRGADecoder::isOpened() const {
return is_opened_;
}
double FFmpegRGADecoder::get(int propId) {
if (propId == cv::CAP_PROP_FRAME_WIDTH)
return (double)width_;
if (propId == cv::CAP_PROP_FRAME_HEIGHT)
return (double)height_;
if (propId == cv::CAP_PROP_FPS)
return fps_;
return 0.0;
}
bool FFmpegRGADecoder::open(const std::string& url) {
cleanUp();
int ret;
AVDictionary* options = nullptr;
// RTSP 优化参数
av_dict_set(&options, "rtsp_transport", "tcp", 0);
av_dict_set(&options, "buffer_size", "1024000", 0);
av_dict_set(&options, "stimeout", "5000000", 0);
av_dict_set(&options, "flags", "low_delay", 0);
spdlog::info("FFmpeg: Opening stream: {}", url);
if ((ret = avformat_open_input(&fmt_ctx_, url.c_str(), nullptr, &options)) < 0) {
spdlog::error("FFmpeg: Could not open input URL: {}", url);
return false;
}
if ((ret = avformat_find_stream_info(fmt_ctx_, nullptr)) < 0)
return false;
video_stream_idx_ = av_find_best_stream(fmt_ctx_, AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0);
if (video_stream_idx_ < 0)
return false;
AVStream* st = fmt_ctx_->streams[video_stream_idx_];
AVCodecParameters* codecpar = st->codecpar;
const char* decoder_name = nullptr;
if (codecpar->codec_id == AV_CODEC_ID_HEVC)
decoder_name = "hevc_rkmpp";
else if (codecpar->codec_id == AV_CODEC_ID_H264)
decoder_name = "h264_rkmpp";
if (decoder_name) {
decoder_ = avcodec_find_decoder_by_name(decoder_name);
if (decoder_)
spdlog::info("FFmpeg: Using Hardware Decoder: {}", decoder_name);
}
if (!decoder_) {
spdlog::warn("FFmpeg: Hardware decoder not found, falling back.");
decoder_ = avcodec_find_decoder(codecpar->codec_id);
}
if (!decoder_)
return false;
dec_ctx_ = avcodec_alloc_context3(decoder_);
avcodec_parameters_to_context(dec_ctx_, codecpar);
if ((ret = avcodec_open2(dec_ctx_, decoder_, nullptr)) < 0) {
spdlog::error("FFmpeg: Failed to open codec");
return false;
}
width_ = codecpar->width;
height_ = codecpar->height;
fps_ = (st->avg_frame_rate.den > 0) ? av_q2d(st->avg_frame_rate) : 25.0;
is_opened_ = true;
spdlog::info("FFmpeg: Decoder Ready. {}x{} @ {:.2f} fps", width_, height_, fps_);
return true;
}
bool FFmpegRGADecoder::ensure_output_buffer(int width, int height) {
size_t img_size = width * height * 3;
if (!output_dma_buf_ || output_dma_buf_->size < img_size || width_ != width ||
height_ != height) {
width_ = width;
height_ = height;
output_dma_buf_ = std::make_unique<DmaBuffer>(img_size, "/dev/dma_heap/system-uncached");
if (output_dma_buf_->fd < 0) {
spdlog::error("FFmpeg: Failed to allocate output DMA buffer!");
return false;
}
spdlog::info("FFmpeg: Allocated Output DMA Buffer (FD: {})", output_dma_buf_->fd);
}
return true;
}
// 获取源信息
bool get_src_info(const AVFrame* frame, RgaSrcInfo& info) {
info.fd = 0;
info.vaddr = nullptr;
info.wstride = frame->width;
info.hstride = frame->height;
info.format = RK_FORMAT_YCbCr_420_SP; // Default NV12
if (frame->format == AV_PIX_FMT_DRM_PRIME && frame->data[0]) {
AVDRMFrameDescriptor* desc = (AVDRMFrameDescriptor*)frame->data[0];
info.fd = desc->objects[0].fd;
// 1. 获取 Pitch (wstride)
info.wstride = desc->layers[0].planes[0].pitch;
// 2. 计算 hstride
if (desc->layers[0].nb_planes >= 2) {
int y_offset = desc->layers[0].planes[0].offset;
int uv_offset = desc->layers[0].planes[1].offset;
info.hstride = (uv_offset - y_offset) / info.wstride;
} else {
// [修改] 如果没有多平面信息，默认假设紧凑排列 (Compact)
// 之前强制对齐到 16 可能导致读取位置错误（跳过了真正的 UV）
// 如果画面变灰，通常是因为 hstride 算大了。
info.hstride = frame->height;
}
return true;
} else if (frame->format == AV_PIX_FMT_NV12) {
info.wstride = frame->linesize[0];
info.vaddr = (void*)frame->data[0];
info.hstride = frame->height;
return true;
}
// [新增] 检测 10bit 格式 (H.265 常见)
else if (frame->format == AV_PIX_FMT_P010) {
info.wstride = frame->linesize[0] / 2; // P010 也是 2 bytes per pixel? 不，stride 是 bytes
// RGA 对 P010 的处理需要特殊 flag，这里暂时回退到 NV12 处理尝试
// 或者直接报错让软件处理
info.wstride = frame->linesize[0];
info.vaddr = (void*)frame->data[0];
info.hstride = frame->height;
info.format = RK_FORMAT_YCbCr_420_SP_10B; // RGA 支持的 10bit 格式
return true;
}
return false;
}
bool FFmpegRGADecoder::read(cv::Mat& output_mat) {
if (!is_opened_)
return false;
static int frame_count = 0;
frame_count++;
int ret;
while (true) {
ret = av_read_frame(fmt_ctx_, pkt_);
if (ret < 0)
return false;
if (pkt_->stream_index == video_stream_idx_) {
ret = avcodec_send_packet(dec_ctx_, pkt_);
if (ret < 0) {
av_packet_unref(pkt_);
continue;
}
ret = avcodec_receive_frame(dec_ctx_, frame_);
if (ret == 0) {
if (!ensure_output_buffer(frame_->width, frame_->height)) {
av_packet_unref(pkt_);
return false;
}
RgaSrcInfo src_info;
bool has_info = get_src_info(frame_, src_info);
// 调试日志：每 100 帧打印一次 Stride 信息，帮你确认计算是否正确
if (frame_count % 100 == 0) {
spdlog::info("Debug Frame: FD={} W={} H={} WStride={} HStride={} Fmt={}",
src_info.fd, frame_->width, frame_->height, src_info.wstride,
src_info.hstride, frame_->format);
}
if (has_info && src_info.fd > 0) {
// --- 硬件路径 ---
rga_buffer_t src_img, dst_img;
rga_buffer_handle_t src_handle, dst_handle;
// Import
size_t src_size = src_info.wstride * src_info.hstride * 3 / 2;
// 如果是 10bit，大小要翻倍? 不，P010 是 2 bytes per pixel
if (src_info.format == RK_FORMAT_YCbCr_420_SP_10B)
src_size *= 2;
src_handle = importbuffer_fd(src_info.fd, src_size);
dst_handle =
importbuffer_fd(output_dma_buf_->fd, output_dma_buf_->actual_alloc_size);
if (src_handle && dst_handle) {
src_img = wrapbuffer_handle(src_handle, frame_->width, frame_->height,
src_info.format);
dst_img = wrapbuffer_handle(dst_handle, width_, height_, RK_FORMAT_BGR_888);
// 设置 Stride
src_img.wstride = src_info.wstride;
src_img.hstride = src_info.hstride;
dst_img.wstride = width_;
dst_img.hstride = height_;
imsetColorSpace(&src_img, IM_YUV_BT709_LIMIT_RANGE);
imsetColorSpace(&dst_img, IM_RGB_FULL);
IM_STATUS status =
imcvtcolor(src_img, dst_img, src_info.format, RK_FORMAT_BGR_888);
releasebuffer_handle(src_handle);
releasebuffer_handle(dst_handle);
if (status == IM_STATUS_SUCCESS) {
output_mat = cv::Mat(height_, width_, CV_8UC3, output_dma_buf_->vaddr);
av_packet_unref(pkt_);
return true;
} else {
spdlog::error("RGA Fail: {}", imStrError(status));
}
}
}
// --- 软件回退 ---
// 如果走到这里，说明硬件路径失败或无 FD
// 简单的软件转换，确保颜色正常
if (frame_->format == AV_PIX_FMT_NV12) {
// 1. 将有效数据拷贝到连续内存 (去除 stride 影响)
cv::Mat mYUV_Continous(frame_->height * 3 / 2, frame_->width, CV_8UC1);
// 拷贝 Y
uint8_t* src_y = frame_->data[0];
uint8_t* dst_y = mYUV_Continous.data;
for (int i = 0; i < frame_->height; i++) {
memcpy(dst_y + i * frame_->width, src_y + i * frame_->linesize[0],
frame_->width);
}
// 拷贝 UV
uint8_t* src_uv = frame_->data[1];
// 如果 data[1] 为空，尝试根据 height 计算偏移
if (!src_uv)
src_uv = src_y + src_info.hstride * src_info.wstride; // 尝试用 hstride
uint8_t* dst_uv = dst_y + frame_->width * frame_->height;
for (int i = 0; i < frame_->height / 2; i++) {
memcpy(dst_uv + i * frame_->width, src_uv + i * frame_->linesize[0],
frame_->width);
}
cv::Mat mBGR(height_, width_, CV_8UC3, output_dma_buf_->vaddr);
cv::cvtColor(mYUV_Continous, mBGR, cv::COLOR_YUV2BGR_NV12);
if (width_ != frame_->width) {
cv::resize(mBGR, mBGR, cv::Size(width_, height_));
}
output_mat = mBGR;
av_packet_unref(pkt_);
return true;
}
}
}
av_packet_unref(pkt_);
}
}
bool FFmpegRGADecoder::read_raw(RgaFrameInfo& output_info) {
if (!is_opened_)
return false;
int ret;
while (true) {
ret = av_read_frame(fmt_ctx_, pkt_);
if (ret < 0)
return false;
if (pkt_->stream_index == video_stream_idx_) {
ret = avcodec_send_packet(dec_ctx_, pkt_);
if (ret < 0) {
av_packet_unref(pkt_);
continue;
}
ret = avcodec_receive_frame(dec_ctx_, frame_);
if (ret == 0) {
// 利用现有的逻辑获取信息
RgaSrcInfo info;
get_src_info(frame_, info);
output_info.fd = info.fd;
output_info.vaddr = info.vaddr;
output_info.width = frame_->width;
output_info.height = frame_->height;
output_info.wstride = info.wstride;
output_info.hstride = info.hstride;
output_info.format = info.format;
// 注意：这里我们返回了 info，但没有释放 AVPacket 或 AVFrame
// 在单线程模型中，下一帧读取会覆盖 frame_。
// 如果是 Zero-Copy，必须确保在 NPU 推理完成前，frame_ 内存不被释放/覆盖。
// 但 rknn_run 是阻塞的，所以这里直接返回是安全的（前提是 infer 结束后才读下一帧）。
av_packet_unref(pkt_);
return true;
}
}
av_packet_unref(pkt_);
}
}

57
src/rknn/ffmpeg_rga_decoder.h Normal file
View File

@ -0,0 +1,57 @@
#pragma once
#include <memory>
#include <mutex>
#include <opencv2/core.hpp>
#include <string>
extern "C" {
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavutil/error.h>
#include <libavutil/hwcontext_drm.h>
#include <libavutil/pixdesc.h>
}
#include "im2d.h"
#include "rga.h"
// [修改] 引用统一的分配器头文件
#include "dma_allocator.hpp"
#include "rknn/rkYolov8.hpp"
class FFmpegRGADecoder {
public:
FFmpegRGADecoder();
~FFmpegRGADecoder();
bool open(const std::string& url);
bool read(cv::Mat& frame);
void release();
double get(int propId);
// [新增] 添加 isOpened 方法
bool isOpened() const;
// 新增接口：获取原始解码信息（用于 Zero-Copy 推理）
bool read_raw(RgaFrameInfo& info);
private:
void cleanUp();
bool get_rga_src_buffer(const AVFrame* frame, rga_buffer_t& src_buf);
bool ensure_output_buffer(int width, int height);
AVFormatContext* fmt_ctx_ = nullptr;
AVCodecContext* dec_ctx_ = nullptr;
const AVCodec* decoder_ = nullptr;
AVPacket* pkt_ = nullptr;
AVFrame* frame_ = nullptr;
int video_stream_idx_ = -1;
bool is_opened_ = false;
std::unique_ptr<DmaBuffer> output_dma_buf_;
int width_ = 0;
int height_ = 0;
double fps_ = 0.0;
};

292
src/rknn/rkYolov8.cc
View File

@ -7,15 +7,17 @@
#include <cstring> #include <cstring>
#include <iostream> #include <iostream>
#include <mutex> #include <mutex>
#include <opencv2/dnn.hpp>
#include <vector> #include <vector>
#include "RgaApi.h" #include "RgaApi.h"
#include "im2d.h"
#include "rga.h" // 命名空间保持一致 (如果头文件里有，这里也要有；如果没有，这里也不要)
// 假设 rkYolov8.hpp 是 rknn_test 命名空间
// namespace rknn_test {
static std::mutex rga_mtx; static std::mutex rga_mtx;
// 辅助：DFL 计算
static void compute_dfl(float* tensor, int dfl_len, float* box) { static void compute_dfl(float* tensor, int dfl_len, float* box) {
for (int b = 0; b < 4; b++) { for (int b = 0; b < 4; b++) {
float exp_t[16]; float exp_t[16];
@ -52,10 +54,17 @@ rkYolov8::rkYolov8(const std::string& model_path, const std::string& label_path,
this->m_class_num = class_num; this->m_class_num = class_num;
this->conf_threshold = 0.45f; this->conf_threshold = 0.45f;
this->nms_threshold = 0.45f; this->nms_threshold = 0.45f;
this->ctx = 0;
} }
rkYolov8::~rkYolov8() { rkYolov8::~rkYolov8() {
if (ctx) {
if (input_mems_[0])
rknn_destroy_mem(ctx, input_mems_[0]);
for (int i = 0; i < io_num.n_output; ++i) {
if (output_mems_[i])
rknn_destroy_mem(ctx, output_mems_[i]);
}
}
if (input_attrs) if (input_attrs)
free(input_attrs); free(input_attrs);
if (output_attrs) if (output_attrs)
@ -95,6 +104,7 @@ int rkYolov8::init(rknn_context* ctx_in, bool is_slave) {
for (int i = 0; i < io_num.n_output; i++) { for (int i = 0; i < io_num.n_output; i++) {
output_attrs[i].index = i; output_attrs[i].index = i;
rknn_query(ctx, RKNN_QUERY_OUTPUT_ATTR, &(output_attrs[i]), sizeof(rknn_tensor_attr)); rknn_query(ctx, RKNN_QUERY_OUTPUT_ATTR, &(output_attrs[i]), sizeof(rknn_tensor_attr));
// 不需要保存 zp 和 scale 了，因为我们让驱动直接输出 float
} }
if (input_attrs[0].fmt == RKNN_TENSOR_NCHW) { if (input_attrs[0].fmt == RKNN_TENSOR_NCHW) {
@ -107,151 +117,196 @@ int rkYolov8::init(rknn_context* ctx_in, bool is_slave) {
channel = input_attrs[0].dims[3]; channel = input_attrs[0].dims[3];
} }
printf("[rkYolov8] Init: %dx%d, Output Num: %d\n", width, height, io_num.n_output); printf("[rkYolov8] Init Zero-Copy: %dx%d, Output Num: %d\n", width, height, io_num.n_output);
// [NPU Input] 640x640 RGBA // 1. 输入内存
input_dma_buf_ = std::make_unique<DmaBuffer>(width, height, RK_FORMAT_RGBA_8888); input_attrs[0].type = RKNN_TENSOR_UINT8;
if (!input_dma_buf_->isValid()) { input_attrs[0].fmt = RKNN_TENSOR_NHWC;
printf("[rkYolov8] Error: Failed to allocate NPU DMA buffer!\n"); input_mems_[0] = rknn_create_mem(ctx, input_attrs[0].size_with_stride);
if (!input_mems_[0])
return -1;
ret = rknn_set_io_mem(ctx, input_mems_[0], &input_attrs[0]);
if (ret < 0)
return -1;
// 2. 输出内存
for (int i = 0; i < io_num.n_output; i++) {
output_mems_[i] = rknn_create_mem(ctx, output_attrs[i].size_with_stride);
ret = rknn_set_io_mem(ctx, output_mems_[i], &output_attrs[i]);
if (ret < 0)
return -1;
}
return 0;
}
int rkYolov8::Preprocess_RGA(rga_buffer_t src, rga_buffer_t dst, LetterBoxInfo& letter_box_info) {
int dst_width = width;
int dst_height = height;
int src_width = src.width;
int src_height = src.height;
float target_wh_ratio = static_cast<float>(dst_width) / dst_height;
float src_wh_ratio = static_cast<float>(src_width) / src_height;
int resize_w, resize_h;
int pad_w = 0, pad_h = 0;
if (src_wh_ratio > target_wh_ratio) {
resize_w = dst_width;
resize_h = (int)((float)dst_width / src_width * src_height);
pad_h = (dst_height - resize_h) / 2;
letter_box_info.hor = false;
letter_box_info.pad = pad_h;
} else {
resize_h = dst_height;
resize_w = (int)((float)dst_height / src_height * src_width);
pad_w = (dst_width - resize_w) / 2;
letter_box_info.hor = true;
letter_box_info.pad = pad_w;
}
letter_box_info.x_pad = pad_w;
letter_box_info.y_pad = pad_h;
letter_box_info.scale = std::min((float)dst_width / src_width, (float)dst_height / src_height);
im_rect src_rect = {0, 0, src_width, src_height};
im_rect dst_rect = {pad_w, pad_h, resize_w, resize_h};
if (input_mems_[0]->virt_addr) {
memset(input_mems_[0]->virt_addr, 114, input_mems_[0]->size);
}
IM_STATUS status = improcess(src, dst, {}, src_rect, dst_rect, {}, IM_SYNC);
if (status != IM_STATUS_SUCCESS) {
return -1; return -1;
} }
return 0; return 0;
} }
detect_result_group_t rkYolov8::infer(const cv::Mat& ori_img) { detect_result_group_t rkYolov8::infer_raw(const RgaFrameInfo& src_info) {
detect_result_group_t detect_result; detect_result_group_t detect_result;
memset(&detect_result, 0, sizeof(detect_result_group_t)); memset(&detect_result, 0, sizeof(detect_result_group_t));
if (ori_img.empty())
return detect_result;
int img_w = ori_img.cols; rga_buffer_t src_img;
int img_h = ori_img.rows; rga_buffer_handle_t src_handle = 0;
rga_buffer_handle_t dst_handle = 0;
// [内存对齐] 驱动要求输入 buffer 必须对齐 // 1. 源句柄
int aligned_w = 2304; if (src_info.fd > 0) {
int aligned_h = (img_h + 15) & ~15; // 1088 size_t src_size = src_info.wstride * src_info.hstride * 3 / 2;
src_handle = importbuffer_fd(src_info.fd, src_size);
if (!src_dma_buf_ || src_dma_buf_->width != aligned_w || src_dma_buf_->height != aligned_h) { if (src_handle == 0)
src_dma_buf_ = std::make_unique<DmaBuffer>(aligned_w, aligned_h, RK_FORMAT_BGR_888);
if (!src_dma_buf_->isValid()) {
printf("[rkYolov8] Fatal: Failed to allocate Source Buffer!\n");
return detect_result; return detect_result;
} src_img = wrapbuffer_handle(src_handle, src_info.width, src_info.height, src_info.format);
} else {
src_img = wrapbuffer_virtualaddr(src_info.vaddr, src_info.width, src_info.height,
src_info.format);
} }
src_img.wstride = src_info.wstride;
src_img.hstride = src_info.hstride;
// 1. CPU Copy // 2. 目标句柄 (NPU Input)
if (src_dma_buf_->vaddr) { size_t dst_size = width * height * 3; // RGB888, 640*640*3
uint8_t* src_ptr = ori_img.data; dst_handle = importbuffer_fd(input_mems_[0]->fd, dst_size);
uint8_t* dst_ptr = (uint8_t*)src_dma_buf_->vaddr; if (dst_handle == 0) {
int row_bytes_src = img_w * 3; if (src_handle)
int row_bytes_dst = aligned_w * 3; releasebuffer_handle(src_handle);
for (int i = 0; i < img_h; i++) { return detect_result;
memcpy(dst_ptr + i * row_bytes_dst, src_ptr + i * row_bytes_src, row_bytes_src);
}
} }
// [关键修正] 目标是 Packed RGB，Stride = Width
rga_buffer_t dst_img = wrapbuffer_handle(dst_handle, width, height, RK_FORMAT_RGB_888);
// 2. Letterbox Params // 3. 预处理
float scale = std::min((float)width / img_w, (float)height / img_h); LetterBoxInfo box_info;
int new_w = (int)(img_w * scale);
int new_h = (int)(img_h * scale);
int pad_w = (width - new_w) / 2;
int pad_h = (height - new_h) / 2;
// =========================================================================
// [模仿参考代码] 结构体初始化流程
// =========================================================================
rga_buffer_t src_img, dst_img;
// 1. 清零 (和参考代码一致)
memset(&src_img, 0, sizeof(src_img));
memset(&dst_img, 0, sizeof(dst_img));
// 2. 包装 FD (替代参考代码的 wrapbuffer_virtualaddr)
// 注意：这里传的是“真实有效”的宽高，不是对齐后的
src_img = wrapbuffer_fd(src_dma_buf_->fd, img_w, img_h, RK_FORMAT_BGR_888);
dst_img = wrapbuffer_fd(input_dma_buf_->fd, width, height, RK_FORMAT_RGBA_8888);
// 3. [关键修正] 覆盖 stride (物理层面的修正)
// wrapbuffer_fd 默认会把 wstride 设为 img_w (1920)，这会导致 Invalid argument
// 我们必须手动把它改成物理对齐后的 2304
src_img.wstride = aligned_w; // 2304
src_img.hstride = aligned_h; // 1088
// 目标图 stride 必须是 640
dst_img.wstride = width;
dst_img.hstride = height;
// 4. 定义裁剪区域 (逻辑层面)
// imcheck/improcess 会根据这个来计算缩放比例
im_rect src_rect = {0, 0, img_w, img_h};
im_rect dst_rect = {pad_w, pad_h, new_w, new_h};
// 5. 清空背景 (NPU 要求 Letterbox 留黑)
if (input_dma_buf_->vaddr)
memset(input_dma_buf_->vaddr, 114, input_dma_buf_->size);
// 6. 执行 (带锁)
{ {
std::lock_guard<std::mutex> lock(rga_mtx); std::lock_guard<std::mutex> lock(rga_mtx);
// 这里必须设置色彩空间，否则 NV12->RGB 颜色会发灰
imsetColorSpace(&src_img, IM_YUV_BT709_LIMIT_RANGE);
imsetColorSpace(&dst_img, IM_RGB_FULL);
// [调试] 打印参数，确保对齐生效 if (Preprocess_RGA(src_img, dst_img, box_info) != 0) {
// printf("RGA Call: SRC[%dx%d stride=%d] -> DST[%dx%d stride=%d]\n", if (src_handle)
// src_img.width, src_img.height, src_img.wstride, releasebuffer_handle(src_handle);
// dst_img.width, dst_img.height, dst_img.wstride); if (dst_handle)
releasebuffer_handle(dst_handle);
// 调用 improcess (自动处理 resize + cvtcolor)
// 这里的用法等同于 imcvtcolor 但支持缩放
IM_STATUS status = improcess(src_img, dst_img, {}, src_rect, dst_rect, {}, IM_SYNC);
if (status != IM_STATUS_SUCCESS) {
printf("[rkYolov8] RGA Fail: %s (Try imresize if this persists)\n", imStrError(status));
// 如果 improcess 失败，尝试降级为 imresize (虽然颜色可能不对，但先确保存活)
// IM_STATUS retry = imresize(src_img, dst_img);
return detect_result; return detect_result;
} }
} }
// 7. NPU 格式转换 (RGBA -> RGB) if (src_handle)
std::vector<uint8_t> npu_rgb_buf(width * height * 3); releasebuffer_handle(src_handle);
if (input_dma_buf_->vaddr) { if (dst_handle)
uint8_t* s = (uint8_t*)input_dma_buf_->vaddr; releasebuffer_handle(dst_handle);
uint8_t* d = npu_rgb_buf.data();
int total = width * height;
for (int i = 0; i < total; i++) {
d[0] = s[0];
d[1] = s[1];
d[2] = s[2];
s += 4;
d += 3;
}
}
// 8. NPU 推理 // 4. 推理
rknn_input inputs[1]; int ret = rknn_run(ctx, nullptr);
memset(inputs, 0, sizeof(inputs)); if (ret < 0)
inputs[0].index = 0; return detect_result;
inputs[0].type = RKNN_TENSOR_UINT8;
inputs[0].size = width * height * 3;
inputs[0].fmt = RKNN_TENSOR_NHWC;
inputs[0].buf = npu_rgb_buf.data();
rknn_inputs_set(ctx, io_num.n_input, inputs);
// 5. 后处理
// 使用 rknn_outputs_get 自动转 float
rknn_output outputs[io_num.n_output]; rknn_output outputs[io_num.n_output];
memset(outputs, 0, sizeof(outputs)); memset(outputs, 0, sizeof(outputs));
for (int i = 0; i < io_num.n_output; i++) for (int i = 0; i < io_num.n_output; i++)
outputs[i].want_float = 1; outputs[i].want_float = 1; // [重要] 驱动代劳反量化
rknn_run(ctx, nullptr); ret = rknn_outputs_get(ctx, io_num.n_output, outputs, nullptr);
rknn_outputs_get(ctx, io_num.n_output, outputs, nullptr); if (ret < 0)
return detect_result;
post_process_v8_dfl(outputs, scale, pad_w, pad_h, &detect_result); post_process_v8_dfl(outputs, box_info.scale, box_info.x_pad, box_info.y_pad, &detect_result);
rknn_outputs_release(ctx, io_num.n_output, outputs); rknn_outputs_release(ctx, io_num.n_output, outputs);
return detect_result; return detect_result;
} }
// ... (post_process_v8_dfl 保持不变) ... detect_result_group_t rkYolov8::infer(const cv::Mat& ori_img) {
if (ori_img.empty())
return {};
int img_w = ori_img.cols;
int img_h = ori_img.rows;
int aligned_w = (img_w + 15) & ~15;
int aligned_h = (img_h + 15) & ~15;
if (!src_dma_buf_ || src_dma_buf_->width != aligned_w || src_dma_buf_->height != aligned_h) {
src_dma_buf_ = std::make_unique<DmaBuffer>(aligned_w, aligned_h, RK_FORMAT_BGR_888);
}
if (src_dma_buf_->vaddr) {
uint8_t* src_ptr = ori_img.data;
uint8_t* dst_ptr = (uint8_t*)src_dma_buf_->vaddr;
int row_bytes = img_w * 3;
if (aligned_w == img_w) {
memcpy(dst_ptr, src_ptr, row_bytes * img_h);
} else {
for (int i = 0; i < img_h; ++i) {
memcpy(dst_ptr + i * aligned_w * 3, src_ptr + i * row_bytes, row_bytes);
}
}
}
RgaFrameInfo info;
info.fd = src_dma_buf_->fd;
info.vaddr = nullptr;
info.width = img_w;
info.height = img_h;
info.wstride = aligned_w;
info.hstride = aligned_h;
info.format = RK_FORMAT_BGR_888;
return infer_raw(info);
}
// [重要修改] 现在直接返回 buf 指针即可，不需要 get_output_float_buffer 手动计算了
// 因为 rknn_outputs_get 已经把 buf 变成了 float*
float* rkYolov8::get_output_float_buffer(int index) {
// 此函数作废，逻辑移入 post_process
return nullptr;
}
void rkYolov8::post_process_v8_dfl(rknn_output* outputs, float scale, int pad_w, int pad_h, void rkYolov8::post_process_v8_dfl(rknn_output* outputs, float scale, int pad_w, int pad_h,
detect_result_group_t* group) { detect_result_group_t* group) {
std::vector<float> filterBoxes; std::vector<float> filterBoxes;
@ -261,8 +316,11 @@ void rkYolov8::post_process_v8_dfl(rknn_output* outputs, float scale, int pad_w,
for (int i = 0; i < 3; i++) { for (int i = 0; i < 3; i++) {
int box_idx = i * output_per_branch; int box_idx = i * output_per_branch;
int cls_idx = i * output_per_branch + 1; int cls_idx = i * output_per_branch + 1;
// [修复] 直接使用 outputs[].buf 作为 float 指针
float* box_tensor = (float*)outputs[box_idx].buf; float* box_tensor = (float*)outputs[box_idx].buf;
float* cls_tensor = (float*)outputs[cls_idx].buf; float* cls_tensor = (float*)outputs[cls_idx].buf;
int grid_h = output_attrs[box_idx].dims[2]; int grid_h = output_attrs[box_idx].dims[2];
int grid_w = output_attrs[box_idx].dims[3]; int grid_w = output_attrs[box_idx].dims[3];
int stride = height / grid_h; int stride = height / grid_h;
@ -329,3 +387,5 @@ void rkYolov8::post_process_v8_dfl(rknn_output* outputs, float scale, int pad_w,
} }
group->count = count; group->count = count;
} }
// } // namespace rknn_test

46
src/rknn/rkYolov8.hpp
View File

@ -6,10 +6,32 @@
#include <string> #include <string>
#include <vector> #include <vector>
#include "im2d.h"
#include "rga.h"
#include "rknn/dma_allocator.hpp" #include "rknn/dma_allocator.hpp"
#include "rknn/postprocess.h" #include "rknn/postprocess.h"
#include "rknn/rknn_api.h" #include "rknn/rknn_api.h"
// RgaFrameInfo 定义
struct RgaFrameInfo {
int fd;
void* vaddr;
int width;
int height;
int wstride;
int hstride;
int format;
};
// LetterBoxInfo 定义
struct LetterBoxInfo {
bool hor;
int pad;
float scale;
int x_pad;
int y_pad;
};
class rkYolov8 { class rkYolov8 {
public: public:
rkYolov8(const std::string& model_path, const std::string& label_path, int class_num); rkYolov8(const std::string& model_path, const std::string& label_path, int class_num);
@ -17,34 +39,44 @@ public:
int init(rknn_context* ctx_in, bool is_slave); int init(rknn_context* ctx_in, bool is_slave);
rknn_context* get_pctx(); rknn_context* get_pctx();
// 零拷贝接口
detect_result_group_t infer_raw(const RgaFrameInfo& src_info);
// 兼容接口
detect_result_group_t infer(const cv::Mat& ori_img); detect_result_group_t infer(const cv::Mat& ori_img);
private: private:
unsigned char* load_model(const char* filename, int* model_size); unsigned char* load_model(const char* filename, int* model_size);
int Preprocess_RGA(rga_buffer_t src, rga_buffer_t dst, LetterBoxInfo& letter_box_info);
// [修复] 补回 rknn_output* 参数，与 .cc 文件保持一致
void post_process_v8_dfl(rknn_output* outputs, float scale, int pad_w, int pad_h, void post_process_v8_dfl(rknn_output* outputs, float scale, int pad_w, int pad_h,
detect_result_group_t* group); detect_result_group_t* group);
float* get_output_float_buffer(int index);
private: private:
std::string model_path; std::string model_path;
std::string m_label_path; std::string m_label_path;
int m_class_num; int m_class_num;
rknn_context ctx; rknn_context ctx = 0;
bool is_slave = false;
unsigned char* model_data = nullptr; unsigned char* model_data = nullptr;
rknn_input_output_num io_num; rknn_input_output_num io_num;
rknn_tensor_attr* input_attrs = nullptr; rknn_tensor_attr* input_attrs = nullptr;
rknn_tensor_attr* output_attrs = nullptr; rknn_tensor_attr* output_attrs = nullptr;
// [NPU 输入] 目标 buffer (640x640) // NPU 内存
std::unique_ptr<DmaBuffer> input_dma_buf_; rknn_tensor_mem* input_mems_[1] = {nullptr};
rknn_tensor_mem* output_mems_[9] = {nullptr};
// [新增] [RGA 输入] 源图像缓存 buffer (1920x1080) // 中间 DMA 缓存 (用于 Mat 兼容)
// 用于将 OpenCV 的虚拟地址转为物理连续内存，解决 RGA 驱动崩溃问题
std::unique_ptr<DmaBuffer> src_dma_buf_; std::unique_ptr<DmaBuffer> src_dma_buf_;
rknn_input inputs[1]; std::vector<int32_t> out_zps;
std::vector<float> out_scales;
int width = 0; int width = 0;
int height = 0; int height = 0;

208
src/rknn/video_service.cc
View File

@ -1,11 +1,17 @@
// video_service.cc (修改后)
#include "video_service.h" #include "video_service.h"
#include <stdio.h> #include <stdio.h>
#include <mutex>
#include "algorithm/HumanDetectionModule.h" #include "algorithm/HumanDetectionModule.h"
#include "im2d.h"
#include "opencv2/imgproc/imgproc.hpp" #include "opencv2/imgproc/imgproc.hpp"
#include "rga.h"
#include "rknn/dma_allocator.hpp" // [新增] 包含 DmaBuffer
#include "rknn/rkYolov8.hpp" // 包含 RgaFrameInfo 定义
#include "spdlog/spdlog.h" #include "spdlog/spdlog.h"
VideoService::VideoService(std::unique_ptr<IAnalysisModule> module, std::string input_url, VideoService::VideoService(std::unique_ptr<IAnalysisModule> module, std::string input_url,
std::string output_rtsp_url, nlohmann::json module_config) std::string output_rtsp_url, nlohmann::json module_config)
: module_(std::move(module)), : module_(std::move(module)),
@ -22,6 +28,7 @@ VideoService::~VideoService() {
if (running_) { if (running_) {
stop(); stop();
} }
// rtsp_dma_buf_ 会被智能指针自动释放
} }
bool VideoService::start() { bool VideoService::start() {
@ -31,66 +38,42 @@ bool VideoService::start() {
} }
spdlog::info("{} Analysis module initialized successfully.", log_prefix_); spdlog::info("{} Analysis module initialized successfully.", log_prefix_);
std::string gst_input_pipeline = "rtspsrc location=" + input_url_ + spdlog::info("{} Try to Open RTSP Stream (H.265/FFmpeg/RGA)...", log_prefix_);
" latency=0 protocols=tcp ! "
"rtph265depay ! "
"h265parse ! "
"mppvideodec format=16 ! "
"videoconvert ! "
"video/x-raw,format=BGR ! "
"appsink";
spdlog::info("Try to Open RTSP Stream"); // 使用新的解码器打开 RTSP URL
capture_.open(gst_input_pipeline, cv::CAP_GSTREAMER); if (!capture_.open(input_url_)) {
spdlog::error("{} Failed to open RTSP stream: {}", log_prefix_, input_url_);
if (!capture_.isOpened()) {
printf("Error: Could not open RTSP stream: %s\n", input_url_.c_str());
return false; return false;
} else {
spdlog::info("RTSP Stream Opened!");
} }
frame_width_ = static_cast<int>(capture_.get(cv::CAP_PROP_FRAME_WIDTH)); // 获取参数
frame_height_ = static_cast<int>(capture_.get(cv::CAP_PROP_FRAME_HEIGHT)); frame_width_ = (int)capture_.get(cv::CAP_PROP_FRAME_WIDTH);
frame_height_ = (int)capture_.get(cv::CAP_PROP_FRAME_HEIGHT);
frame_fps_ = capture_.get(cv::CAP_PROP_FPS); frame_fps_ = capture_.get(cv::CAP_PROP_FPS);
if (frame_fps_ <= 0) if (frame_fps_ <= 0)
frame_fps_ = 25.0; frame_fps_ = 25.0;
if (frame_width_ == 0 || frame_height_ == 0) { // -----------------------------------------------------------------
spdlog::error( // [关键修改] 初始化 RTSP 推流专用的 DMA Buffer
"{} Failed to get valid frame width or height from GStreamer " // -----------------------------------------------------------------
"pipeline (got {}x{}).", // 宽和高向上对齐到 16，防止 RGA 写入越界
log_prefix_, frame_width_, frame_height_); int aligned_w = (frame_width_ + 15) & (~15);
spdlog::error( int aligned_h = (frame_height_ + 15) & (~15);
"{} This usually means the RTSP stream is unavailable or the "
"GStreamer input pipeline (mppvideodec?) failed.",
log_prefix_);
cv::Mat test_frame; // 申请物理连续内存 (BGR格式)
if (capture_.read(test_frame) && !test_frame.empty()) { rtsp_dma_buf_ = std::make_unique<DmaBuffer>(aligned_w, aligned_h, RK_FORMAT_BGR_888);
frame_width_ = test_frame.cols;
frame_height_ = test_frame.rows;
spdlog::info("{} Successfully got frame size by reading first frame: {}x{}",
log_prefix_, frame_width_, frame_height_);
{ if (!rtsp_dma_buf_->isValid()) {
std::lock_guard<std::mutex> lock(frame_mutex_); spdlog::error("{} Failed to allocate RTSP DMA Buffer!", log_prefix_);
latest_frame_ = test_frame; return false;
new_frame_available_ = true;
}
frame_cv_.notify_one();
} else {
spdlog::error("{} Failed to read first frame to determine size. Aborting.",
log_prefix_);
capture_.release();
return false;
}
} }
spdlog::info("{} Allocated RTSP Buffer (FD: {})", log_prefix_, rtsp_dma_buf_->fd);
printf("RTSP stream opened successfully! (%dx%d @ %.2f FPS)\n", frame_width_, frame_height_, spdlog::info("{} Stream Opened! {}x{} @ {:.2f} FPS", log_prefix_, frame_width_, frame_height_,
frame_fps_); frame_fps_);
// GStreamer Pipeline
std::string gst_pipeline = std::string gst_pipeline =
"appsrc ! " "appsrc ! "
"queue max-size-buffers=2 leaky=downstream ! " "queue max-size-buffers=2 leaky=downstream ! "
@ -121,11 +104,11 @@ bool VideoService::start() {
return true; return true;
} }
bool VideoService::set_analysis_mode(int mode_val) { bool VideoService::set_analysis_mode(int mode_val) {
if (!module_) if (!module_)
return false; return false;
// 检查是否为 HumanDetectionModule
auto* human_module = dynamic_cast<HumanDetectionModule*>(module_.get()); auto* human_module = dynamic_cast<HumanDetectionModule*>(module_.get());
if (human_module) { if (human_module) {
@ -143,91 +126,152 @@ bool VideoService::set_analysis_mode(int mode_val) {
spdlog::warn("{} Module is not HumanDetectionModule, cannot set mode.", log_prefix_); spdlog::warn("{} Module is not HumanDetectionModule, cannot set mode.", log_prefix_);
return false; return false;
} }
return false; // [修复 3] 添加这个缺少 return 的补丁
} }
void VideoService::stop() { void VideoService::stop() {
printf("Stopping VideoService...\n"); printf("Stopping VideoService...\n");
running_ = false; running_ = false;
frame_cv_.notify_all(); frame_cv_.notify_all();
if (reading_thread_.joinable()) { if (reading_thread_.joinable())
reading_thread_.join(); reading_thread_.join();
} if (processing_thread_.joinable())
if (processing_thread_.joinable()) {
processing_thread_.join(); processing_thread_.join();
}
printf("Processing thread joined.\n"); printf("Processing thread joined.\n");
if (capture_.isOpened()) { if (capture_.isOpened())
capture_.release(); capture_.release();
} if (writer_.isOpened())
if (writer_.isOpened()) {
writer_.release(); writer_.release();
}
module_->stop(); module_->stop();
module_.reset(); module_.reset();
rtsp_dma_buf_.reset(); // [新增] 释放 DMA buffer
printf("VideoService stopped.\n"); printf("VideoService stopped.\n");
} }
void VideoService::reading_loop() { void VideoService::reading_loop() {
cv::Mat frame; // 使用全局的 RgaFrameInfo (如果不使用 namespace rknn_test)
RgaFrameInfo info;
spdlog::info("Reading thread started."); spdlog::info("Reading thread started.");
while (running_) { while (running_) {
if (!capture_.read(frame)) { // [修改] 调用 read_raw 获取原始 FD 和 Stride
spdlog::warn("Reading loop: Failed to read frame from capture. Stopping service."); if (!capture_.read_raw(info)) {
spdlog::warn("Reading loop: Failed to read frame or stream EOF.");
running_ = false; running_ = false;
break; break;
} }
if (frame.empty()) {
continue;
}
{ {
std::lock_guard<std::mutex> lock(frame_mutex_); std::lock_guard<std::mutex> lock(frame_mutex_);
latest_frame_ = frame; // 存储最新的原始帧信息
latest_frame_info_ = info;
new_frame_available_ = true; new_frame_available_ = true;
} }
frame_cv_.notify_one(); frame_cv_.notify_one();
} }
frame_cv_.notify_all(); // 确保 processing_loop 也会退出 frame_cv_.notify_all();
spdlog::info("Reading loop finished."); spdlog::info("Reading loop finished.");
} }
void VideoService::processing_loop() { void VideoService::processing_loop() {
cv::Mat frame; RgaFrameInfo current_info;
cv::Mat display_frame;
while (running_) { while (running_) {
{ {
// 1. (不变) 获取帧
std::unique_lock<std::mutex> lock(frame_mutex_); std::unique_lock<std::mutex> lock(frame_mutex_);
frame_cv_.wait(lock, [&] { return new_frame_available_ || !running_; }); frame_cv_.wait(lock, [&] { return new_frame_available_ || !running_; });
if (!running_) { if (!running_)
break; break;
}
frame = latest_frame_.clone(); current_info = latest_frame_info_;
new_frame_available_ = false; new_frame_available_ = false;
} }
if (frame.empty()) { // =========================================================
// [核心逻辑] 使用 RGA 将 NV12(FD) 转为 BGR(FD)
// =========================================================
bool rga_success = false;
// 只有当源 FD 有效且目标 Buffer 就绪时才执行硬件转换
if (current_info.fd > 0 && rtsp_dma_buf_ && rtsp_dma_buf_->isValid()) {
rga_buffer_t src_img, dst_img;
memset(&src_img, 0, sizeof(src_img));
memset(&dst_img, 0, sizeof(dst_img));
// 1. 设置源 (NV12 from Decoder)
src_img = wrapbuffer_fd(current_info.fd, current_info.width, current_info.height,
current_info.format);
src_img.wstride = current_info.wstride; // [关键] 设置正确的 stride
src_img.hstride = current_info.hstride; // [关键] 设置正确的 stride
// 2. 设置目标 (BGR for RTSP/Analysis)
dst_img =
wrapbuffer_fd(rtsp_dma_buf_->fd, frame_width_, frame_height_, RK_FORMAT_BGR_888);
// 目标 buffer 是紧凑的，或者按照 alloc 的对齐来
dst_img.wstride = rtsp_dma_buf_->width; // 这里取 alloc 时的 aligned_w
dst_img.hstride = rtsp_dma_buf_->height;
// 3. 执行转换 (NV12 -> BGR)
// 使用 improcess 可以同时处理裁剪和格式转换
im_rect src_rect = {0, 0, current_info.width, current_info.height};
im_rect dst_rect = {0, 0, frame_width_, frame_height_};
// 设置色彩空间 (BT.709 Limit -> RGB Full) 解决发灰问题
imsetColorSpace(&src_img, IM_YUV_BT709_LIMIT_RANGE);
imsetColorSpace(&dst_img, IM_RGB_FULL);
IM_STATUS status = improcess(src_img, dst_img, {}, src_rect, dst_rect, {}, IM_SYNC);
if (status == IM_STATUS_SUCCESS) {
// 转换成功，构造一个指向该 DMA 内存的 Mat
// 注意：Mat 的 step 必须匹配 buffer 的 stride
display_frame = cv::Mat(frame_height_, frame_width_, CV_8UC3, rtsp_dma_buf_->vaddr,
rtsp_dma_buf_->width * 3);
rga_success = true;
} else {
spdlog::warn("{} RGA Conversion Failed: {}", log_prefix_, imStrError(status));
}
}
// =========================================================
// [软件回退] 如果 RGA 失败或无 FD，使用 CPU 转换
// =========================================================
if (!rga_success) {
if (current_info.vaddr && current_info.format == RK_FORMAT_YCbCr_420_SP) {
// 简易软转逻辑 (NV12 -> BGR)
// 为了防止 stride 问题，这里简化处理，假设是紧凑的
// 实际生产中应参考 ffmpeg_rga_decoder.cpp 中的逐行拷贝逻辑
cv::Mat mYUV(current_info.height * 3 / 2, current_info.width, CV_8UC1,
current_info.vaddr);
cv::cvtColor(mYUV, display_frame, cv::COLOR_YUV2BGR_NV12);
} else {
continue; // 无法处理，跳过
}
}
if (display_frame.empty())
continue; continue;
// ---------------------------------------------------------
// 推理与推流
// ---------------------------------------------------------
// 将转换好的 BGR Mat 传给算法模块
// 这里的 display_frame 数据位于 DMA Buffer 中，对于后续可能的 RGA 操作是友好的
if (!module_->process(display_frame)) {
// spdlog::warn("{} Module failed to process frame.", log_prefix_);
} }
if (!module_->process(frame)) {
// 模块报告处理失败
spdlog::warn("{} Module failed to process frame. Skipping.", log_prefix_);
}
if (writer_.isOpened()) { if (writer_.isOpened()) {
writer_.write(frame); writer_.write(display_frame);
} }
} }

8
src/rknn/video_service.h
View File

@ -12,6 +12,7 @@
#include <thread> #include <thread>
#include "algorithm/IAnalysisModule.h" #include "algorithm/IAnalysisModule.h"
#include "ffmpeg_rga_decoder.h"
#include "nlohmann/json.hpp" #include "nlohmann/json.hpp"
class VideoService { class VideoService {
@ -42,7 +43,8 @@ private:
int frame_width_ = 0; int frame_width_ = 0;
int frame_height_ = 0; int frame_height_ = 0;
double frame_fps_ = 0.0; double frame_fps_ = 0.0;
cv::VideoCapture capture_; // cv::VideoCapture capture_;
FFmpegRGADecoder capture_;
cv::VideoWriter writer_; cv::VideoWriter writer_;
std::thread processing_thread_; std::thread processing_thread_;
@ -54,4 +56,8 @@ private:
bool new_frame_available_{false}; bool new_frame_available_{false};
std::string log_prefix_; std::string log_prefix_;
// [新增] 用于存储从解码器获取的最新帧信息
RgaFrameInfo latest_frame_info_; // 存储原始帧信息
std::unique_ptr<DmaBuffer> rtsp_dma_buf_; // 存储转换后的BGR数据
}; };

53
src/rknn_test/DmaBuffer.h
View File

@ -1,53 +0,0 @@
#include <fcntl.h>
#include <linux/dma-heap.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
#include <unistd.h>
#include <iostream>
#include <string>
class DmaBuffer {
public:
int fd;
void* vaddr; // 虚拟地址（仅供调试或CPU必须介入时使用，如画字）
size_t size;
DmaBuffer(size_t size, const std::string& heap_name = "/dev/dma_heap/system") {
this->size = size;
this->vaddr = MAP_FAILED;
this->fd = -1;
int heap_fd = open(heap_name.c_str(), O_RDONLY | O_CLOEXEC);
if (heap_fd < 0) {
perror("Failed to open dma heap");
return;
}
struct dma_heap_allocation_data data = {0};
data.len = size;
data.fd_flags = O_RDWR | O_CLOEXEC;
if (ioctl(heap_fd, DMA_HEAP_IOCTL_ALLOC, &data) < 0) {
perror("dma heap alloc failed");
close(heap_fd);
return;
}
this->fd = data.fd;
close(heap_fd);
// 如果需要CPU访问（例如写OSD文字），则映射，否则可以不映射
this->vaddr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, this->fd, 0);
if (this->vaddr == MAP_FAILED) {
perror("mmap failed");
}
}
~DmaBuffer() {
if (vaddr != MAP_FAILED)
munmap(vaddr, size);
if (fd >= 0)
close(fd);
}
};