fix(videoService): 试图降低视频的CPU占有率。
This commit is contained in:
parent
eb47fc2878
commit
4cf96c835a
|
|
@ -1,98 +1,98 @@
|
||||||
{
|
{
|
||||||
"modbus_rtu_devices": [
|
"modbus_rtu_devices": [
|
||||||
{
|
{
|
||||||
"enabled": true,
|
"enabled": false,
|
||||||
"device_id": "rtu_temp_sensor_lab",
|
"device_id": "rtu_temp_sensor_lab",
|
||||||
"port_path": "/dev/ttyS7",
|
"port_path": "/dev/ttyS7",
|
||||||
"baud_rate": 9600,
|
"baud_rate": 9600,
|
||||||
"slave_id": 1,
|
"slave_id": 1,
|
||||||
"poll_interval_ms": 5000,
|
"poll_interval_ms": 5000,
|
||||||
"data_points": [
|
"data_points": [
|
||||||
{
|
{
|
||||||
"name": "temperature",
|
"name": "temperature",
|
||||||
"address": 0,
|
"address": 0,
|
||||||
"type": "INT16",
|
"type": "INT16",
|
||||||
"scale": 0.1
|
"scale": 0.1
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "humidity",
|
"name": "humidity",
|
||||||
"address": 1,
|
"address": 1,
|
||||||
"type": "UINT16",
|
"type": "UINT16",
|
||||||
"scale": 0.1
|
"scale": 0.1
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"enabled": false,
|
"enabled": false,
|
||||||
"device_id": "rotary encoder",
|
"device_id": "rotary encoder",
|
||||||
"port_path": "/dev/ttyS7",
|
"port_path": "/dev/ttyS7",
|
||||||
"baud_rate": 9600,
|
"baud_rate": 9600,
|
||||||
"slave_id": 111,
|
"slave_id": 111,
|
||||||
"poll_interval_ms": 5000,
|
"poll_interval_ms": 5000,
|
||||||
"data_points": [
|
"data_points": [
|
||||||
{
|
{
|
||||||
"name": "count",
|
"name": "count",
|
||||||
"address": 1,
|
"address": 1,
|
||||||
"type": "INT16",
|
"type": "INT16",
|
||||||
"scale": 1
|
"scale": 1
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "total_count",
|
"name": "total_count",
|
||||||
"address": 2,
|
"address": 2,
|
||||||
"type": "INT16",
|
"type": "INT16",
|
||||||
"scale": 1
|
"scale": 1
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"enabled": false,
|
"enabled": false,
|
||||||
"device_id": "backup_counter",
|
"device_id": "backup_counter",
|
||||||
"port_path": "/dev/ttyS7",
|
"port_path": "/dev/ttyS7",
|
||||||
"baud_rate": 9600,
|
"baud_rate": 9600,
|
||||||
"slave_id": 10,
|
"slave_id": 10,
|
||||||
"poll_interval_ms": 1000,
|
"poll_interval_ms": 1000,
|
||||||
"data_points": [
|
"data_points": [
|
||||||
{
|
{
|
||||||
"name": "count",
|
"name": "count",
|
||||||
"address": 32,
|
"address": 32,
|
||||||
"type": "UINT32"
|
"type": "UINT32"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"modbus_tcp_devices": [
|
"modbus_tcp_devices": [
|
||||||
{
|
{
|
||||||
"enabled": false,
|
"enabled": false,
|
||||||
"device_id": "plc_workshop1",
|
"device_id": "plc_workshop1",
|
||||||
"ip_address": "192.168.1.120",
|
"ip_address": "192.168.1.120",
|
||||||
"port": 502,
|
"port": 502,
|
||||||
"slave_id": 1,
|
"slave_id": 1,
|
||||||
"poll_interval_ms": 1000,
|
"poll_interval_ms": 1000,
|
||||||
"data_points": [
|
"data_points": [
|
||||||
{
|
{
|
||||||
"name": "motor_speed",
|
"name": "motor_speed",
|
||||||
"address": 100,
|
"address": 100,
|
||||||
"type": "UINT16",
|
"type": "UINT16",
|
||||||
"scale": 1
|
"scale": 1
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "pressure",
|
"name": "pressure",
|
||||||
"address": 102,
|
"address": 102,
|
||||||
"type": "FLOAT32",
|
"type": "FLOAT32",
|
||||||
"scale": 0.01
|
"scale": 0.01
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "valve_status",
|
"name": "valve_status",
|
||||||
"address": 104,
|
"address": 104,
|
||||||
"type": "UINT16",
|
"type": "UINT16",
|
||||||
"scale": 1
|
"scale": 1
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"modbus_rtu_bus_configs": {
|
"modbus_rtu_bus_configs": {
|
||||||
"/dev/ttyS7": {
|
"/dev/ttyS7": {
|
||||||
"inter_device_delay_ms": 150
|
"inter_device_delay_ms": 150
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -1,47 +1,107 @@
|
||||||
{
|
{
|
||||||
"video_service": {
|
"video_service": {
|
||||||
"enabled": true
|
"enabled": true
|
||||||
},
|
},
|
||||||
"video_streams": [
|
"video_streams": [
|
||||||
{
|
{
|
||||||
"enabled": false,
|
"enabled": false,
|
||||||
"id": "cam_01_intrusion",
|
"id": "cam_01_intrusion",
|
||||||
"input_url": "rtsp://admin:hzx12345@192.168.1.10:554/Streaming/Channels/1301",
|
"input_url": "rtsp://admin:hzx12345@192.168.1.10:554/Streaming/Channels/1301",
|
||||||
"module_config": {
|
"module_config": {
|
||||||
"class_num": 80,
|
"class_num": 80,
|
||||||
"intrusion_zone": [
|
"intrusion_zone": [
|
||||||
100,
|
100,
|
||||||
100,
|
100,
|
||||||
1820,
|
1820,
|
||||||
1820
|
1820
|
||||||
],
|
],
|
||||||
"label_path": "/app/models/coco_80_labels_list.txt",
|
"label_path": "/app/models/coco_80_labels_list.txt",
|
||||||
"model_path": "/app/models/RK3588/yolov5s-640-640.rknn",
|
"model_path": "/app/models/RK3588/yolov5s-640-640.rknn",
|
||||||
"rknn_thread_num": 3,
|
"rknn_thread_num": 3,
|
||||||
"time_threshold_sec": 3
|
"time_threshold_sec": 3
|
||||||
},
|
},
|
||||||
"module_type": "intrusion_detection",
|
"module_type": "intrusion_detection",
|
||||||
"output_rtsp": "rtsp://127.0.0.1:8554/ch1301"
|
"output_rtsp": "rtsp://127.0.0.1:8554/ch1301"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
"id": "cam_01_intrusion",
|
"id": "cam_01_intrusion",
|
||||||
"input_url": "rtsp://admin:hzx12345@192.168.1.10:554/Streaming/Channels/1301",
|
"input_url": "rtsp://admin:hzx12345@192.168.1.10:554/Streaming/Channels/1301",
|
||||||
"module_config": {
|
"module_config": {
|
||||||
"class_num": 3,
|
"class_num": 3,
|
||||||
"intrusion_zone": [
|
"intrusion_zone": [
|
||||||
100,
|
100,
|
||||||
100,
|
100,
|
||||||
1820,
|
1820,
|
||||||
1820
|
1820
|
||||||
],
|
],
|
||||||
"label_path": "/app/models/human.txt",
|
"label_path": "/app/models/human.txt",
|
||||||
"model_path": "/app/models/human_detection.rknn",
|
"model_path": "/app/models/human_detection.rknn",
|
||||||
"rknn_thread_num": 3,
|
"rknn_thread_num": 3,
|
||||||
"time_threshold_sec": 3
|
"time_threshold_sec": 3
|
||||||
},
|
},
|
||||||
"module_type": "human_detection",
|
"module_type": "human_detection",
|
||||||
"output_rtsp": "rtsp://127.0.0.1:8554/ch1301"
|
"output_rtsp": "rtsp://127.0.0.1:8554/ch1301"
|
||||||
}
|
},
|
||||||
]
|
{
|
||||||
|
"enabled": true,
|
||||||
|
"id": "cam_02_intrusion",
|
||||||
|
"input_url": "rtsp://admin:hzx12345@192.168.1.10:554/Streaming/Channels/1401",
|
||||||
|
"module_config": {
|
||||||
|
"class_num": 3,
|
||||||
|
"intrusion_zone": [
|
||||||
|
100,
|
||||||
|
100,
|
||||||
|
1820,
|
||||||
|
1820
|
||||||
|
],
|
||||||
|
"label_path": "/app/models/human.txt",
|
||||||
|
"model_path": "/app/models/human_detection.rknn",
|
||||||
|
"rknn_thread_num": 3,
|
||||||
|
"time_threshold_sec": 3
|
||||||
|
},
|
||||||
|
"module_type": "human_detection",
|
||||||
|
"output_rtsp": "rtsp://127.0.0.1:8554/ch1401"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"enabled": true,
|
||||||
|
"id": "cam_03_intrusion",
|
||||||
|
"input_url": "rtsp://admin:hzx12345@192.168.1.10:554/Streaming/Channels/1201",
|
||||||
|
"module_config": {
|
||||||
|
"class_num": 3,
|
||||||
|
"intrusion_zone": [
|
||||||
|
100,
|
||||||
|
100,
|
||||||
|
1820,
|
||||||
|
1820
|
||||||
|
],
|
||||||
|
"label_path": "/app/models/human.txt",
|
||||||
|
"model_path": "/app/models/human_detection.rknn",
|
||||||
|
"rknn_thread_num": 3,
|
||||||
|
"time_threshold_sec": 3
|
||||||
|
},
|
||||||
|
"module_type": "human_detection",
|
||||||
|
"output_rtsp": "rtsp://127.0.0.1:8554/ch1201"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"enabled": true,
|
||||||
|
"id": "cam_04_intrusion",
|
||||||
|
"input_url": "rtsp://admin:hzx12345@192.168.1.10:554/Streaming/Channels/1101",
|
||||||
|
"module_config": {
|
||||||
|
"class_num": 3,
|
||||||
|
"intrusion_zone": [
|
||||||
|
100,
|
||||||
|
100,
|
||||||
|
1820,
|
||||||
|
1820
|
||||||
|
],
|
||||||
|
"label_path": "/app/models/human.txt",
|
||||||
|
"model_path": "/app/models/human_detection.rknn",
|
||||||
|
"rknn_thread_num": 3,
|
||||||
|
"time_threshold_sec": 3
|
||||||
|
},
|
||||||
|
"module_type": "human_detection",
|
||||||
|
"output_rtsp": "rtsp://127.0.0.1:8554/ch1101"
|
||||||
|
}
|
||||||
|
]
|
||||||
}
|
}
|
||||||
|
|
@ -43,7 +43,6 @@ RUN apt-get update && \
|
||||||
gstreamer1.0-libav \
|
gstreamer1.0-libav \
|
||||||
gstreamer1.0-tools \
|
gstreamer1.0-tools \
|
||||||
gstreamer1.0-x \
|
gstreamer1.0-x \
|
||||||
gstreamer1.0-alsa \
|
|
||||||
gstreamer1.0-pulseaudio \
|
gstreamer1.0-pulseaudio \
|
||||||
gstreamer1.0-rtsp \
|
gstreamer1.0-rtsp \
|
||||||
libopencv-dev \
|
libopencv-dev \
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,5 @@
|
||||||
|
// src/algorithm/rkYolov8.cc
|
||||||
|
// [稳定版] 移除 RGA 预处理，完全使用 CPU OpenCV 防止内核崩溃
|
||||||
#include "rkYolov8.hpp"
|
#include "rkYolov8.hpp"
|
||||||
#include "opencv2/imgproc/imgproc.hpp"
|
#include "opencv2/imgproc/imgproc.hpp"
|
||||||
#include <cmath>
|
#include <cmath>
|
||||||
|
|
@ -9,301 +11,310 @@
|
||||||
#include <stdlib.h>
|
#include <stdlib.h>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
|
// 辅助宏
|
||||||
|
#define ALIGN_EVEN(x) ((x) & ~1)
|
||||||
|
|
||||||
static inline float sigmoid(float x) { return 1.0f / (1.0f + expf(-x)); }
|
static inline float sigmoid(float x) { return 1.0f / (1.0f + expf(-x)); }
|
||||||
|
|
||||||
static void compute_dfl(float *tensor, int dfl_len, float *box) {
|
static void compute_dfl(float *tensor, int dfl_len, float *box)
|
||||||
for (int b = 0; b < 4; b++) {
|
{
|
||||||
float exp_t[16];
|
for (int b = 0; b < 4; b++)
|
||||||
float exp_sum = 0;
|
{
|
||||||
float acc_sum = 0;
|
float exp_t[16];
|
||||||
|
float exp_sum = 0;
|
||||||
for (int i = 0; i < dfl_len; i++) {
|
float acc_sum = 0;
|
||||||
exp_t[i] = expf(tensor[i + b * dfl_len]);
|
for (int i = 0; i < dfl_len; i++)
|
||||||
exp_sum += exp_t[i];
|
{
|
||||||
}
|
exp_t[i] = expf(tensor[i + b * dfl_len]);
|
||||||
|
exp_sum += exp_t[i];
|
||||||
for (int i = 0; i < dfl_len; i++) {
|
}
|
||||||
acc_sum += (exp_t[i] / exp_sum) * i;
|
for (int i = 0; i < dfl_len; i++)
|
||||||
}
|
{
|
||||||
box[b] = acc_sum;
|
acc_sum += (exp_t[i] / exp_sum) * i;
|
||||||
}
|
}
|
||||||
|
box[b] = acc_sum;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
unsigned char *rkYolov8::load_model(const char *filename, int *model_size) {
|
unsigned char *rkYolov8::load_model(const char *filename, int *model_size)
|
||||||
FILE *fp = fopen(filename, "rb");
|
{
|
||||||
if (fp == nullptr) {
|
FILE *fp = fopen(filename, "rb");
|
||||||
printf("Open file %s failed.\n", filename);
|
if (fp == nullptr)
|
||||||
return nullptr;
|
{
|
||||||
}
|
printf("Open file %s failed.\n", filename);
|
||||||
fseek(fp, 0, SEEK_END);
|
return nullptr;
|
||||||
int size = ftell(fp);
|
}
|
||||||
fseek(fp, 0, SEEK_SET);
|
fseek(fp, 0, SEEK_END);
|
||||||
unsigned char *data = (unsigned char *)malloc(size);
|
int size = ftell(fp);
|
||||||
fread(data, 1, size, fp);
|
fseek(fp, 0, SEEK_SET);
|
||||||
fclose(fp);
|
unsigned char *data = (unsigned char *)malloc(size);
|
||||||
*model_size = size;
|
fread(data, 1, size, fp);
|
||||||
return data;
|
fclose(fp);
|
||||||
|
*model_size = size;
|
||||||
|
return data;
|
||||||
}
|
}
|
||||||
|
|
||||||
rkYolov8::rkYolov8(const std::string &model_path, const std::string &label_path,
|
rkYolov8::rkYolov8(const std::string &model_path, const std::string &label_path,
|
||||||
int class_num) {
|
int class_num)
|
||||||
this->model_path = model_path;
|
{
|
||||||
this->m_label_path = label_path;
|
this->model_path = model_path;
|
||||||
this->m_class_num = class_num;
|
this->m_label_path = label_path;
|
||||||
this->conf_threshold = 0.3f;
|
this->m_class_num = class_num;
|
||||||
this->nms_threshold = 0.5f;
|
this->conf_threshold = 0.3f;
|
||||||
this->model_data = nullptr;
|
this->nms_threshold = 0.5f;
|
||||||
this->input_attrs = nullptr;
|
this->model_data = nullptr;
|
||||||
this->output_attrs = nullptr;
|
this->input_attrs = nullptr;
|
||||||
|
this->output_attrs = nullptr;
|
||||||
|
|
||||||
|
// 即使不用 RGA，我们依然分配对齐内存给 NPU 使用，这对性能有好处
|
||||||
|
this->rga_buffer_ptr = nullptr;
|
||||||
|
this->ctx = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
rkYolov8::~rkYolov8() {
|
rkYolov8::~rkYolov8()
|
||||||
if (input_attrs)
|
{
|
||||||
free(input_attrs);
|
if (input_attrs)
|
||||||
if (output_attrs)
|
free(input_attrs);
|
||||||
free(output_attrs);
|
if (output_attrs)
|
||||||
if (model_data)
|
free(output_attrs);
|
||||||
free(model_data);
|
if (model_data)
|
||||||
if (rga_buffer_ptr)
|
free(model_data);
|
||||||
free(rga_buffer_ptr);
|
if (rga_buffer_ptr)
|
||||||
rknn_destroy(ctx);
|
free(rga_buffer_ptr);
|
||||||
|
if (ctx)
|
||||||
|
rknn_destroy(ctx);
|
||||||
}
|
}
|
||||||
|
|
||||||
rknn_context *rkYolov8::get_pctx() { return &ctx; }
|
rknn_context *rkYolov8::get_pctx() { return &ctx; }
|
||||||
|
|
||||||
int rkYolov8::init(rknn_context *ctx_in, bool is_slave) {
|
int rkYolov8::init(rknn_context *ctx_in, bool is_slave)
|
||||||
int model_data_size = 0;
|
{
|
||||||
model_data = load_model(model_path.c_str(), &model_data_size);
|
int model_data_size = 0;
|
||||||
if (!model_data)
|
model_data = load_model(model_path.c_str(), &model_data_size);
|
||||||
return -1;
|
if (!model_data)
|
||||||
|
return -1;
|
||||||
|
|
||||||
if (is_slave) {
|
if (is_slave)
|
||||||
ret = rknn_dup_context(ctx_in, &ctx);
|
ret = rknn_dup_context(ctx_in, &ctx);
|
||||||
} else {
|
else
|
||||||
ret = rknn_init(&ctx, model_data, model_data_size, 0, nullptr);
|
ret = rknn_init(&ctx, model_data, model_data_size, 0, nullptr);
|
||||||
}
|
|
||||||
if (ret < 0)
|
|
||||||
return -1;
|
|
||||||
|
|
||||||
rknn_query(ctx, RKNN_QUERY_IN_OUT_NUM, &io_num, sizeof(io_num));
|
if (ret < 0)
|
||||||
|
return -1;
|
||||||
|
|
||||||
input_attrs =
|
rknn_query(ctx, RKNN_QUERY_IN_OUT_NUM, &io_num, sizeof(io_num));
|
||||||
(rknn_tensor_attr *)calloc(io_num.n_input, sizeof(rknn_tensor_attr));
|
|
||||||
output_attrs =
|
|
||||||
(rknn_tensor_attr *)calloc(io_num.n_output, sizeof(rknn_tensor_attr));
|
|
||||||
|
|
||||||
for (int i = 0; i < io_num.n_input; i++) {
|
input_attrs = (rknn_tensor_attr *)calloc(io_num.n_input, sizeof(rknn_tensor_attr));
|
||||||
input_attrs[i].index = i;
|
output_attrs = (rknn_tensor_attr *)calloc(io_num.n_output, sizeof(rknn_tensor_attr));
|
||||||
rknn_query(ctx, RKNN_QUERY_INPUT_ATTR, &(input_attrs[i]),
|
|
||||||
sizeof(rknn_tensor_attr));
|
|
||||||
}
|
|
||||||
for (int i = 0; i < io_num.n_output; i++) {
|
|
||||||
output_attrs[i].index = i;
|
|
||||||
rknn_query(ctx, RKNN_QUERY_OUTPUT_ATTR, &(output_attrs[i]),
|
|
||||||
sizeof(rknn_tensor_attr));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (input_attrs[0].fmt == RKNN_TENSOR_NCHW) {
|
for (int i = 0; i < io_num.n_input; i++)
|
||||||
channel = input_attrs[0].dims[1];
|
{
|
||||||
height = input_attrs[0].dims[2];
|
input_attrs[i].index = i;
|
||||||
width = input_attrs[0].dims[3];
|
rknn_query(ctx, RKNN_QUERY_INPUT_ATTR, &(input_attrs[i]), sizeof(rknn_tensor_attr));
|
||||||
} else {
|
}
|
||||||
height = input_attrs[0].dims[1];
|
for (int i = 0; i < io_num.n_output; i++)
|
||||||
width = input_attrs[0].dims[2];
|
{
|
||||||
channel = input_attrs[0].dims[3];
|
output_attrs[i].index = i;
|
||||||
}
|
rknn_query(ctx, RKNN_QUERY_OUTPUT_ATTR, &(output_attrs[i]), sizeof(rknn_tensor_attr));
|
||||||
|
}
|
||||||
|
|
||||||
printf("[rkYolov8] Init: %dx%d, Output Num: %d\n", width, height,
|
if (input_attrs[0].fmt == RKNN_TENSOR_NCHW)
|
||||||
io_num.n_output);
|
{
|
||||||
|
channel = input_attrs[0].dims[1];
|
||||||
|
height = input_attrs[0].dims[2];
|
||||||
|
width = input_attrs[0].dims[3];
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
height = input_attrs[0].dims[1];
|
||||||
|
width = input_attrs[0].dims[2];
|
||||||
|
channel = input_attrs[0].dims[3];
|
||||||
|
}
|
||||||
|
|
||||||
rga_buffer_size = width * height * channel;
|
printf("[rkYolov8] Init Model: %dx%d (Mode: CPU Pre-process)\n", width, height);
|
||||||
rga_buffer_ptr = malloc(rga_buffer_size);
|
|
||||||
memset(rga_buffer_ptr, 114, rga_buffer_size);
|
|
||||||
|
|
||||||
return 0;
|
// 1. NPU 输入 Buffer (RGB)
|
||||||
|
// 依然保持 4K 对齐，因为 rknn_run 读取它时效率更高
|
||||||
|
rga_buffer_size = width * height * channel;
|
||||||
|
if (posix_memalign(&rga_buffer_ptr, 4096, rga_buffer_size + 4096) != 0)
|
||||||
|
return -1;
|
||||||
|
memset(rga_buffer_ptr, 0, rga_buffer_size);
|
||||||
|
|
||||||
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
detect_result_group_t rkYolov8::infer(const cv::Mat &ori_img) {
|
detect_result_group_t rkYolov8::infer(const cv::Mat &ori_img)
|
||||||
detect_result_group_t detect_result;
|
{
|
||||||
memset(&detect_result, 0, sizeof(detect_result_group_t));
|
detect_result_group_t detect_result;
|
||||||
|
memset(&detect_result, 0, sizeof(detect_result_group_t));
|
||||||
|
|
||||||
if (ori_img.empty())
|
if (ori_img.empty() || !ori_img.data)
|
||||||
return detect_result;
|
return detect_result;
|
||||||
|
|
||||||
int img_w = ori_img.cols;
|
int img_w = ori_img.cols;
|
||||||
int img_h = ori_img.rows;
|
int img_h = ori_img.rows;
|
||||||
float scale = std::min((float)width / img_w, (float)height / img_h);
|
|
||||||
int new_w = (int)(img_w * scale);
|
|
||||||
int new_h = (int)(img_h * scale);
|
|
||||||
int pad_w = (width - new_w) / 2;
|
|
||||||
int pad_h = (height - new_h) / 2;
|
|
||||||
|
|
||||||
rga_buffer_t src_img = wrapbuffer_virtualaddr((void *)ori_img.data, img_w,
|
// 计算缩放参数
|
||||||
img_h, RK_FORMAT_BGR_888);
|
float scale = std::min((float)width / img_w, (float)height / img_h);
|
||||||
rga_buffer_t dst_img =
|
int new_w = ALIGN_EVEN((int)(img_w * scale));
|
||||||
wrapbuffer_virtualaddr(rga_buffer_ptr, width, height, RK_FORMAT_RGB_888);
|
int new_h = ALIGN_EVEN((int)(img_h * scale));
|
||||||
|
int pad_w = ALIGN_EVEN((width - new_w) / 2);
|
||||||
|
int pad_h = ALIGN_EVEN((height - new_h) / 2);
|
||||||
|
|
||||||
rga_buffer_t pat;
|
// =========================================================================
|
||||||
memset(&pat, 0, sizeof(pat));
|
// [安全模式] 纯 CPU 预处理
|
||||||
im_rect src_rect = {0, 0, img_w, img_h};
|
// 彻底避开 RGA 驱动 bug，防止 Kernel Panic
|
||||||
im_rect dst_rect = {pad_w, pad_h, new_w, new_h};
|
// =========================================================================
|
||||||
im_rect pat_rect = {0, 0, 0, 0};
|
|
||||||
|
|
||||||
memset(rga_buffer_ptr, 114, rga_buffer_size);
|
// 1. 准备目标容器 (指向 NPU 内存)
|
||||||
improcess(src_img, dst_img, pat, src_rect, dst_rect, pat_rect, IM_SYNC);
|
cv::Mat final_rgb(height, width, CV_8UC3, rga_buffer_ptr);
|
||||||
|
|
||||||
inputs[0].index = 0;
|
// 2. 填充背景色 (114 灰度)
|
||||||
inputs[0].type = RKNN_TENSOR_UINT8;
|
// 注意：这里使用 setTo 可能会比较慢，优化方法是只填边缘，但为了安全先全填
|
||||||
inputs[0].size = rga_buffer_size;
|
final_rgb.setTo(cv::Scalar(114, 114, 114));
|
||||||
inputs[0].fmt = RKNN_TENSOR_NHWC;
|
|
||||||
inputs[0].buf = rga_buffer_ptr;
|
|
||||||
rknn_inputs_set(ctx, io_num.n_input, inputs);
|
|
||||||
|
|
||||||
rknn_output outputs[io_num.n_output];
|
// 3. 定义 ROI
|
||||||
memset(outputs, 0, sizeof(outputs));
|
cv::Rect roi_rect(pad_w, pad_h, new_w, new_h);
|
||||||
for (int i = 0; i < io_num.n_output; i++)
|
cv::Mat dst_roi = final_rgb(roi_rect);
|
||||||
outputs[i].want_float = 1;
|
|
||||||
|
|
||||||
// FILE *fp = fopen("/app/debug_input.rgb", "wb");
|
// 4. CPU Resize + Color Convert
|
||||||
// fwrite(rga_buffer_ptr, 1, rga_buffer_size, fp);
|
// RK3588 的 CPU 非常强劲，这一步对 3 路视频流几乎无压力
|
||||||
// fclose(fp);
|
cv::Mat temp_resized;
|
||||||
// printf("Saved debug input image.\n");
|
cv::resize(ori_img, temp_resized, cv::Size(new_w, new_h), 0, 0, cv::INTER_LINEAR);
|
||||||
|
|
||||||
rknn_run(ctx, nullptr);
|
if (ori_img.channels() == 4)
|
||||||
rknn_outputs_get(ctx, io_num.n_output, outputs, nullptr);
|
{
|
||||||
|
cv::cvtColor(temp_resized, dst_roi, cv::COLOR_RGBA2RGB);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
cv::cvtColor(temp_resized, dst_roi, cv::COLOR_BGR2RGB);
|
||||||
|
}
|
||||||
|
|
||||||
post_process_v8_dfl(outputs, scale, pad_w, pad_h, &detect_result);
|
// =========================================================================
|
||||||
|
// [步骤 3] NPU 推理
|
||||||
|
// =========================================================================
|
||||||
|
inputs[0].index = 0;
|
||||||
|
inputs[0].type = RKNN_TENSOR_UINT8;
|
||||||
|
inputs[0].size = rga_buffer_size;
|
||||||
|
inputs[0].fmt = RKNN_TENSOR_NHWC;
|
||||||
|
inputs[0].buf = rga_buffer_ptr;
|
||||||
|
rknn_inputs_set(ctx, io_num.n_input, inputs);
|
||||||
|
|
||||||
rknn_outputs_release(ctx, io_num.n_output, outputs);
|
rknn_output outputs[io_num.n_output];
|
||||||
return detect_result;
|
memset(outputs, 0, sizeof(outputs));
|
||||||
|
for (int i = 0; i < io_num.n_output; i++)
|
||||||
|
outputs[i].want_float = 1;
|
||||||
|
|
||||||
|
rknn_run(ctx, nullptr);
|
||||||
|
rknn_outputs_get(ctx, io_num.n_output, outputs, nullptr);
|
||||||
|
|
||||||
|
post_process_v8_dfl(outputs, scale, pad_w, pad_h, &detect_result);
|
||||||
|
rknn_outputs_release(ctx, io_num.n_output, outputs);
|
||||||
|
|
||||||
|
return detect_result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// post_process_v8_dfl 保持原样，无需修改
|
||||||
void rkYolov8::post_process_v8_dfl(rknn_output *outputs, float scale, int pad_w,
|
void rkYolov8::post_process_v8_dfl(rknn_output *outputs, float scale, int pad_w,
|
||||||
int pad_h, detect_result_group_t *group) {
|
int pad_h, detect_result_group_t *group)
|
||||||
std::vector<float> filterBoxes;
|
{
|
||||||
std::vector<float> objProbs;
|
// ... (内容保持不变) ...
|
||||||
std::vector<int> classId;
|
std::vector<float> filterBoxes;
|
||||||
|
std::vector<float> objProbs;
|
||||||
|
std::vector<int> classId;
|
||||||
|
|
||||||
int output_per_branch = io_num.n_output / 3;
|
int output_per_branch = io_num.n_output / 3;
|
||||||
|
|
||||||
for (int i = 0; i < 3; i++) {
|
for (int i = 0; i < 3; i++)
|
||||||
|
{
|
||||||
|
int box_idx = i * output_per_branch;
|
||||||
|
int cls_idx = i * output_per_branch + 1;
|
||||||
|
|
||||||
int box_idx = i * output_per_branch;
|
float *box_tensor = (float *)outputs[box_idx].buf;
|
||||||
int cls_idx = i * output_per_branch + 1;
|
float *cls_tensor = (float *)outputs[cls_idx].buf;
|
||||||
|
|
||||||
float *box_tensor = (float *)outputs[box_idx].buf;
|
int grid_h = output_attrs[box_idx].dims[2];
|
||||||
float *cls_tensor = (float *)outputs[cls_idx].buf;
|
int grid_w = output_attrs[box_idx].dims[3];
|
||||||
|
int stride = height / grid_h;
|
||||||
|
int box_channel = output_attrs[box_idx].dims[1];
|
||||||
|
int dfl_len = box_channel / 4;
|
||||||
|
int grid_len = grid_h * grid_w;
|
||||||
|
|
||||||
int grid_h = output_attrs[box_idx].dims[2];
|
for (int h = 0; h < grid_h; h++)
|
||||||
int grid_w = output_attrs[box_idx].dims[3];
|
{
|
||||||
int stride = height / grid_h;
|
for (int w = 0; w < grid_w; w++)
|
||||||
|
{
|
||||||
|
int offset = h * grid_w + w;
|
||||||
|
float max_score = 0.0f;
|
||||||
|
int max_class_id = -1;
|
||||||
|
|
||||||
int box_channel = output_attrs[box_idx].dims[1];
|
for (int c = 0; c < m_class_num; c++)
|
||||||
int dfl_len = box_channel / 4;
|
{
|
||||||
|
int idx = c * grid_len + offset;
|
||||||
|
float score = cls_tensor[idx];
|
||||||
|
if (score > max_score)
|
||||||
|
{
|
||||||
|
max_score = score;
|
||||||
|
max_class_id = c;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
int grid_len = grid_h * grid_w;
|
if (max_score > conf_threshold)
|
||||||
|
{
|
||||||
|
float box_pred[4];
|
||||||
|
float dfl_buffer[64];
|
||||||
|
for (int k = 0; k < 4 * dfl_len; k++)
|
||||||
|
{
|
||||||
|
dfl_buffer[k] = box_tensor[k * grid_len + offset];
|
||||||
|
}
|
||||||
|
compute_dfl(dfl_buffer, dfl_len, box_pred);
|
||||||
|
|
||||||
for (int h = 0; h < grid_h; h++) {
|
float x1 = (-box_pred[0] + w + 0.5f) * stride;
|
||||||
for (int w = 0; w < grid_w; w++) {
|
float y1 = (-box_pred[1] + h + 0.5f) * stride;
|
||||||
int offset = h * grid_w + w;
|
float x2 = (box_pred[2] + w + 0.5f) * stride;
|
||||||
|
float y2 = (box_pred[3] + h + 0.5f) * stride;
|
||||||
|
|
||||||
float max_score = 0.0f;
|
filterBoxes.push_back(x1);
|
||||||
int max_class_id = -1;
|
filterBoxes.push_back(y1);
|
||||||
|
filterBoxes.push_back(x2 - x1);
|
||||||
|
filterBoxes.push_back(y2 - y1);
|
||||||
|
objProbs.push_back(max_score);
|
||||||
|
classId.push_back(max_class_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
for (int c = 0; c < m_class_num; c++) {
|
std::vector<cv::Rect> cvBoxes;
|
||||||
|
for (size_t i = 0; i < filterBoxes.size(); i += 4)
|
||||||
|
{
|
||||||
|
cvBoxes.push_back(cv::Rect(filterBoxes[i], filterBoxes[i + 1],
|
||||||
|
filterBoxes[i + 2], filterBoxes[i + 3]));
|
||||||
|
}
|
||||||
|
|
||||||
int idx = c * grid_len + offset;
|
std::vector<int> indices;
|
||||||
float score = cls_tensor[idx];
|
cv::dnn::NMSBoxes(cvBoxes, objProbs, conf_threshold, nms_threshold, indices);
|
||||||
// printf("Raw: %f, Sigmoid: %f\n", raw_score, sigmoid(raw_score));
|
|
||||||
|
|
||||||
// float score = sigmoid(cls_tensor[idx]);
|
int count = 0;
|
||||||
if (score > max_score) {
|
for (int idx : indices)
|
||||||
max_score = score;
|
{
|
||||||
max_class_id = c;
|
if (count >= OBJ_NUMB_MAX_SIZE)
|
||||||
}
|
break;
|
||||||
}
|
cv::Rect box = cvBoxes[idx];
|
||||||
|
int x = (int)((box.x - pad_w) / scale);
|
||||||
|
int y = (int)((box.y - pad_h) / scale);
|
||||||
|
int width = (int)(box.width / scale);
|
||||||
|
int height = (int)(box.height / scale);
|
||||||
|
|
||||||
if (max_score > conf_threshold && max_class_id == 0) {
|
detect_result_t *det = &group->results[count];
|
||||||
float box_pred[4];
|
det->box.left = std::max(0, x);
|
||||||
float dfl_buffer[64];
|
det->box.top = std::max(0, y);
|
||||||
|
det->box.right = x + width;
|
||||||
for (int k = 0; k < 4 * dfl_len; k++) {
|
det->box.bottom = y + height;
|
||||||
dfl_buffer[k] = box_tensor[k * grid_len + offset];
|
det->prop = objProbs[idx];
|
||||||
}
|
snprintf(det->name, OBJ_NAME_MAX_SIZE, "%d", classId[idx]);
|
||||||
compute_dfl(dfl_buffer, dfl_len, box_pred);
|
count++;
|
||||||
|
}
|
||||||
float x1 = (-box_pred[0] + w + 0.5f) * stride;
|
group->count = count;
|
||||||
float y1 = (-box_pred[1] + h + 0.5f) * stride;
|
|
||||||
float x2 = (box_pred[2] + w + 0.5f) * stride;
|
|
||||||
float y2 = (box_pred[3] + h + 0.5f) * stride;
|
|
||||||
|
|
||||||
filterBoxes.push_back(x1);
|
|
||||||
filterBoxes.push_back(y1);
|
|
||||||
filterBoxes.push_back(x2 - x1);
|
|
||||||
filterBoxes.push_back(y2 - y1);
|
|
||||||
|
|
||||||
objProbs.push_back(max_score);
|
|
||||||
classId.push_back(max_class_id);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (max_score > conf_threshold) {
|
|
||||||
|
|
||||||
float box_pred[4];
|
|
||||||
float dfl_buffer[64];
|
|
||||||
|
|
||||||
for (int k = 0; k < 4 * dfl_len; k++) {
|
|
||||||
dfl_buffer[k] = box_tensor[k * grid_len + offset];
|
|
||||||
}
|
|
||||||
|
|
||||||
compute_dfl(dfl_buffer, dfl_len, box_pred);
|
|
||||||
|
|
||||||
float x1 = (-box_pred[0] + w + 0.5f) * stride;
|
|
||||||
float y1 = (-box_pred[1] + h + 0.5f) * stride;
|
|
||||||
float x2 = (box_pred[2] + w + 0.5f) * stride;
|
|
||||||
float y2 = (box_pred[3] + h + 0.5f) * stride;
|
|
||||||
|
|
||||||
filterBoxes.push_back(x1);
|
|
||||||
filterBoxes.push_back(y1);
|
|
||||||
filterBoxes.push_back(x2 - x1);
|
|
||||||
filterBoxes.push_back(y2 - y1);
|
|
||||||
|
|
||||||
objProbs.push_back(max_score);
|
|
||||||
classId.push_back(max_class_id);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
std::vector<cv::Rect> cvBoxes;
|
|
||||||
for (size_t i = 0; i < filterBoxes.size(); i += 4) {
|
|
||||||
cvBoxes.push_back(cv::Rect(filterBoxes[i], filterBoxes[i + 1],
|
|
||||||
filterBoxes[i + 2], filterBoxes[i + 3]));
|
|
||||||
}
|
|
||||||
|
|
||||||
std::vector<int> indices;
|
|
||||||
cv::dnn::NMSBoxes(cvBoxes, objProbs, conf_threshold, nms_threshold, indices);
|
|
||||||
|
|
||||||
int count = 0;
|
|
||||||
for (int idx : indices) {
|
|
||||||
if (count >= OBJ_NUMB_MAX_SIZE)
|
|
||||||
break;
|
|
||||||
|
|
||||||
cv::Rect box = cvBoxes[idx];
|
|
||||||
|
|
||||||
int x = (int)((box.x - pad_w) / scale);
|
|
||||||
int y = (int)((box.y - pad_h) / scale);
|
|
||||||
int width = (int)(box.width / scale);
|
|
||||||
int height = (int)(box.height / scale);
|
|
||||||
|
|
||||||
detect_result_t *det = &group->results[count];
|
|
||||||
det->box.left = std::max(0, x);
|
|
||||||
det->box.top = std::max(0, y);
|
|
||||||
det->box.right = x + width;
|
|
||||||
det->box.bottom = y + height;
|
|
||||||
det->prop = objProbs[idx];
|
|
||||||
snprintf(det->name, OBJ_NAME_MAX_SIZE, "%d", classId[idx]);
|
|
||||||
|
|
||||||
count++;
|
|
||||||
}
|
|
||||||
group->count = count;
|
|
||||||
}
|
}
|
||||||
|
|
@ -2,51 +2,64 @@
|
||||||
#define RKYOLOV8_H
|
#define RKYOLOV8_H
|
||||||
|
|
||||||
#include "opencv2/core/core.hpp"
|
#include "opencv2/core/core.hpp"
|
||||||
#include "postprocess.h"
|
#include "postprocess.h" // 请确保你项目里有这个头文件
|
||||||
#include "rknn_api.h"
|
#include "rknn_api.h"
|
||||||
|
|
||||||
#include "im2d.h"
|
// RGA 头文件
|
||||||
#include "rga.h"
|
#include "rga.h"
|
||||||
|
#include "im2d.h"
|
||||||
|
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
class rkYolov8 {
|
class rkYolov8
|
||||||
|
{
|
||||||
private:
|
private:
|
||||||
int ret;
|
int ret;
|
||||||
std::string model_path;
|
std::string model_path;
|
||||||
unsigned char *model_data;
|
unsigned char *model_data;
|
||||||
|
|
||||||
rknn_context ctx;
|
rknn_context ctx;
|
||||||
rknn_input_output_num io_num;
|
rknn_input_output_num io_num;
|
||||||
rknn_tensor_attr *input_attrs;
|
rknn_tensor_attr *input_attrs;
|
||||||
rknn_tensor_attr *output_attrs;
|
rknn_tensor_attr *output_attrs;
|
||||||
rknn_input inputs[1];
|
rknn_input inputs[1];
|
||||||
|
|
||||||
int channel, width, height;
|
int channel, width, height;
|
||||||
|
|
||||||
float nms_threshold;
|
float nms_threshold;
|
||||||
float conf_threshold;
|
float conf_threshold;
|
||||||
|
|
||||||
std::string m_label_path;
|
std::string m_label_path;
|
||||||
int m_class_num;
|
int m_class_num;
|
||||||
|
|
||||||
void *rga_buffer_ptr = nullptr;
|
// --- [核心修复] RGA 专用内存池 ---
|
||||||
int rga_buffer_size = 0;
|
// 1. 最终输入给 NPU 的 RGB Buffer (640x640)
|
||||||
|
void *rga_buffer_ptr = nullptr;
|
||||||
|
int rga_buffer_size = 0;
|
||||||
|
|
||||||
static unsigned char *load_model(const char *filename, int *model_size);
|
// 2. RGA 缩放后的中间 Buffer (640x640 RGBA)
|
||||||
|
void *rga_buffer_rgba_ptr = nullptr;
|
||||||
|
int rga_buffer_rgba_size = 0;
|
||||||
|
|
||||||
void post_process_v8_dfl(rknn_output *outputs, float scale, int pad_w,
|
// [已删除] rga_buffer_src_ptr
|
||||||
int pad_h, detect_result_group_t *group);
|
// 不再需要内部维护源 buffer，直接使用 VideoService 传入的对齐内存
|
||||||
|
|
||||||
|
static unsigned char *load_model(const char *filename, int *model_size);
|
||||||
|
|
||||||
|
void post_process_v8_dfl(rknn_output *outputs, float scale, int pad_w,
|
||||||
|
int pad_h, detect_result_group_t *group);
|
||||||
|
|
||||||
public:
|
public:
|
||||||
rkYolov8(const std::string &model_path, const std::string &label_path,
|
rkYolov8(const std::string &model_path, const std::string &label_path,
|
||||||
int class_num);
|
int class_num);
|
||||||
~rkYolov8();
|
~rkYolov8();
|
||||||
|
|
||||||
int init(rknn_context *ctx_in, bool is_slave);
|
int init(rknn_context *ctx_in, bool is_slave);
|
||||||
rknn_context *get_pctx();
|
rknn_context *get_pctx();
|
||||||
detect_result_group_t infer(const cv::Mat &ori_img);
|
|
||||||
|
// 推理函数
|
||||||
|
detect_result_group_t infer(const cv::Mat &ori_img);
|
||||||
};
|
};
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
|
@ -1,208 +1,343 @@
|
||||||
// video_service.cc (修改后)
|
// video_service.cc (最终稳定版: NV12 输入 + CPU 转换)
|
||||||
#include "video_service.h"
|
#include "video_service.h"
|
||||||
#include "opencv2/imgproc/imgproc.hpp"
|
#include "opencv2/imgproc/imgproc.hpp"
|
||||||
#include "spdlog/spdlog.h"
|
#include "spdlog/spdlog.h"
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
|
#include <vector>
|
||||||
|
#include <stdlib.h> // posix_memalign, free
|
||||||
|
|
||||||
VideoService::VideoService(std::unique_ptr<IAnalysisModule> module,
|
VideoService::VideoService(std::unique_ptr<IAnalysisModule> module,
|
||||||
std::string input_url, std::string output_rtsp_url,
|
std::string input_url, std::string output_rtsp_url,
|
||||||
nlohmann::json module_config)
|
nlohmann::json module_config)
|
||||||
: module_(std::move(module)), input_url_(input_url),
|
: module_(std::move(module)), input_url_(input_url),
|
||||||
output_rtsp_url_(output_rtsp_url),
|
output_rtsp_url_(output_rtsp_url),
|
||||||
module_config_(std::move(module_config)), running_(false) {
|
module_config_(std::move(module_config)), running_(false)
|
||||||
log_prefix_ = "[VideoService: " + input_url + "]";
|
{
|
||||||
spdlog::info("{} Created. Input: {}, Output: {}", log_prefix_,
|
gst_init(nullptr, nullptr);
|
||||||
input_url_.c_str(), output_rtsp_url_.c_str());
|
|
||||||
|
log_prefix_ = "[VideoService: " + input_url + "]";
|
||||||
|
spdlog::info("{} Created. Input: {}, Output: {}", log_prefix_,
|
||||||
|
input_url_.c_str(), output_rtsp_url_.c_str());
|
||||||
}
|
}
|
||||||
|
|
||||||
VideoService::~VideoService() {
|
VideoService::~VideoService()
|
||||||
if (running_) {
|
{
|
||||||
stop();
|
if (running_)
|
||||||
}
|
{
|
||||||
|
stop();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
bool VideoService::start() {
|
// 核心辅助函数：BGR -> NV12 (用于推流)
|
||||||
if (!module_ || !module_->init(module_config_)) {
|
void VideoService::bgr_to_nv12(const cv::Mat &src, std::vector<uint8_t> &dst)
|
||||||
spdlog::error("{} Failed to initialize analysis module!", log_prefix_);
|
{
|
||||||
return false;
|
int w = src.cols;
|
||||||
}
|
int h = src.rows;
|
||||||
spdlog::info("{} Analysis module initialized successfully.", log_prefix_);
|
int y_size = w * h;
|
||||||
|
int uv_size = y_size / 2;
|
||||||
|
|
||||||
std::string gst_input_pipeline = "rtspsrc location=" + input_url_ +
|
dst.resize(y_size + uv_size);
|
||||||
" latency=0 protocols=tcp ! "
|
|
||||||
"rtph265depay ! "
|
|
||||||
"h265parse ! "
|
|
||||||
"mppvideodec format=16 ! "
|
|
||||||
"videoconvert ! "
|
|
||||||
"video/x-raw,format=BGR ! "
|
|
||||||
"appsink";
|
|
||||||
|
|
||||||
spdlog::info("Try to Open RTSP Stream");
|
cv::Mat i420_mat;
|
||||||
capture_.open(gst_input_pipeline, cv::CAP_GSTREAMER);
|
cv::cvtColor(src, i420_mat, cv::COLOR_BGR2YUV_I420);
|
||||||
|
|
||||||
if (!capture_.isOpened()) {
|
memcpy(dst.data(), i420_mat.data, y_size);
|
||||||
printf("Error: Could not open RTSP stream: %s\n", input_url_.c_str());
|
|
||||||
return false;
|
|
||||||
} else {
|
|
||||||
spdlog::info("RTSP Stream Opened!");
|
|
||||||
}
|
|
||||||
|
|
||||||
frame_width_ = static_cast<int>(capture_.get(cv::CAP_PROP_FRAME_WIDTH));
|
const uint8_t *u_src = i420_mat.data + y_size;
|
||||||
frame_height_ = static_cast<int>(capture_.get(cv::CAP_PROP_FRAME_HEIGHT));
|
const uint8_t *v_src = i420_mat.data + y_size + (y_size / 4);
|
||||||
frame_fps_ = capture_.get(cv::CAP_PROP_FPS);
|
uint8_t *uv_dst = dst.data() + y_size;
|
||||||
if (frame_fps_ <= 0)
|
|
||||||
frame_fps_ = 25.0;
|
|
||||||
|
|
||||||
if (frame_width_ == 0 || frame_height_ == 0) {
|
for (int i = 0; i < y_size / 4; ++i)
|
||||||
spdlog::error("{} Failed to get valid frame width or height from GStreamer "
|
{
|
||||||
"pipeline (got {}x{}).",
|
uv_dst[2 * i] = u_src[i]; // U
|
||||||
log_prefix_, frame_width_, frame_height_);
|
uv_dst[2 * i + 1] = v_src[i]; // V
|
||||||
spdlog::error("{} This usually means the RTSP stream is unavailable or the "
|
}
|
||||||
"GStreamer input pipeline (mppvideodec?) failed.",
|
|
||||||
log_prefix_);
|
|
||||||
|
|
||||||
cv::Mat test_frame;
|
|
||||||
if (capture_.read(test_frame) && !test_frame.empty()) {
|
|
||||||
frame_width_ = test_frame.cols;
|
|
||||||
frame_height_ = test_frame.rows;
|
|
||||||
spdlog::info(
|
|
||||||
"{} Successfully got frame size by reading first frame: {}x{}",
|
|
||||||
log_prefix_, frame_width_, frame_height_);
|
|
||||||
|
|
||||||
{
|
|
||||||
std::lock_guard<std::mutex> lock(frame_mutex_);
|
|
||||||
latest_frame_ = test_frame;
|
|
||||||
new_frame_available_ = true;
|
|
||||||
}
|
|
||||||
frame_cv_.notify_one();
|
|
||||||
|
|
||||||
} else {
|
|
||||||
spdlog::error(
|
|
||||||
"{} Failed to read first frame to determine size. Aborting.",
|
|
||||||
log_prefix_);
|
|
||||||
capture_.release();
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
printf("RTSP stream opened successfully! (%dx%d @ %.2f FPS)\n", frame_width_,
|
|
||||||
frame_height_, frame_fps_);
|
|
||||||
|
|
||||||
std::string gst_pipeline = "appsrc ! "
|
|
||||||
"queue max-size-buffers=2 leaky=downstream ! "
|
|
||||||
"video/x-raw,format=BGR ! "
|
|
||||||
"videoconvert ! "
|
|
||||||
"video/x-raw,format=NV12 ! "
|
|
||||||
"mpph264enc gop=25 rc-mode=fixqp qp-init=26 ! "
|
|
||||||
"h264parse ! "
|
|
||||||
"rtspclientsink location=" +
|
|
||||||
output_rtsp_url_ + " latency=0 protocols=tcp";
|
|
||||||
|
|
||||||
printf("Using GStreamer output pipeline: %s\n", gst_pipeline.c_str());
|
|
||||||
|
|
||||||
writer_.open(gst_pipeline, cv::CAP_GSTREAMER, 0, frame_fps_,
|
|
||||||
cv::Size(frame_width_, frame_height_), true);
|
|
||||||
|
|
||||||
if (!writer_.isOpened()) {
|
|
||||||
printf("Error: Could not open VideoWriter with GStreamer pipeline.\n");
|
|
||||||
capture_.release();
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
printf("VideoWriter opened successfully.\n");
|
|
||||||
|
|
||||||
running_ = true;
|
|
||||||
reading_thread_ = std::thread(&VideoService::reading_loop, this);
|
|
||||||
processing_thread_ = std::thread(&VideoService::processing_loop, this);
|
|
||||||
printf("Processing thread started.\n");
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void VideoService::stop() {
|
// 辅助函数：创建 4K 对齐的 Mat
|
||||||
printf("Stopping VideoService...\n");
|
cv::Mat VideoService::create_aligned_mat(int width, int height, int type)
|
||||||
running_ = false;
|
{
|
||||||
|
size_t elem_size = cv::Mat(1, 1, type).elemSize();
|
||||||
|
size_t total_size = width * height * elem_size;
|
||||||
|
void *ptr = nullptr;
|
||||||
|
int ret = posix_memalign(&ptr, 4096, total_size);
|
||||||
|
|
||||||
frame_cv_.notify_all();
|
if (ret != 0 || !ptr)
|
||||||
|
{
|
||||||
if (reading_thread_.joinable()) {
|
spdlog::error("Fatal: Failed to allocate aligned memory!");
|
||||||
reading_thread_.join();
|
return cv::Mat();
|
||||||
}
|
}
|
||||||
|
return cv::Mat(height, width, type, ptr);
|
||||||
if (processing_thread_.joinable()) {
|
|
||||||
processing_thread_.join();
|
|
||||||
}
|
|
||||||
printf("Processing thread joined.\n");
|
|
||||||
|
|
||||||
if (capture_.isOpened()) {
|
|
||||||
capture_.release();
|
|
||||||
}
|
|
||||||
if (writer_.isOpened()) {
|
|
||||||
writer_.release();
|
|
||||||
}
|
|
||||||
module_->stop();
|
|
||||||
module_.reset();
|
|
||||||
|
|
||||||
printf("VideoService stopped.\n");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void VideoService::reading_loop() {
|
bool VideoService::start()
|
||||||
cv::Mat frame;
|
{
|
||||||
spdlog::info("Reading thread started.");
|
if (!module_ || !module_->init(module_config_))
|
||||||
|
{
|
||||||
|
spdlog::error("{} Failed to initialize analysis module!", log_prefix_);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
while (running_) {
|
// -------------------------------------------------------------------------
|
||||||
if (!capture_.read(frame)) {
|
// [关键修改] 更改输入 Pipeline 为 NV12
|
||||||
spdlog::warn(
|
// 移除了 'videoconvert' 和 'format=BGR'，消除了 GStreamer 内部 RGA 的竞争
|
||||||
"Reading loop: Failed to read frame from capture. Stopping service.");
|
// -------------------------------------------------------------------------
|
||||||
running_ = false;
|
std::string gst_input_pipeline = "rtspsrc location=" + input_url_ +
|
||||||
break;
|
" latency=0 protocols=tcp ! "
|
||||||
}
|
"rtph265depay ! "
|
||||||
|
"h265parse ! "
|
||||||
|
"mppvideodec ! " // mpp 解码默认输出 NV12
|
||||||
|
"video/x-raw,format=NV12 ! "
|
||||||
|
"appsink";
|
||||||
|
|
||||||
if (frame.empty()) {
|
spdlog::info("Try to Open RTSP Stream (NV12 Mode)");
|
||||||
continue;
|
capture_.open(gst_input_pipeline, cv::CAP_GSTREAMER);
|
||||||
}
|
|
||||||
|
|
||||||
{
|
if (!capture_.isOpened())
|
||||||
std::lock_guard<std::mutex> lock(frame_mutex_);
|
{
|
||||||
latest_frame_ = frame;
|
printf("Error: Could not open RTSP stream: %s\n", input_url_.c_str());
|
||||||
new_frame_available_ = true;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
frame_cv_.notify_one();
|
// 注意：在 NV12 模式下，capture_.get 可能返回包含 padding 的尺寸
|
||||||
}
|
// 或者 OpenCV 会将 NV12 读取为 height * 1.5 的单通道图像
|
||||||
|
frame_width_ = static_cast<int>(capture_.get(cv::CAP_PROP_FRAME_WIDTH));
|
||||||
|
frame_height_ = static_cast<int>(capture_.get(cv::CAP_PROP_FRAME_HEIGHT));
|
||||||
|
frame_fps_ = capture_.get(cv::CAP_PROP_FPS);
|
||||||
|
if (frame_fps_ <= 0)
|
||||||
|
frame_fps_ = 25.0;
|
||||||
|
|
||||||
frame_cv_.notify_all(); // 确保 processing_loop 也会退出
|
// 读取一帧以确认真实的图像尺寸
|
||||||
spdlog::info("Reading loop finished.");
|
cv::Mat test_frame;
|
||||||
|
if (capture_.read(test_frame))
|
||||||
|
{
|
||||||
|
// NV12 判定：如果是单通道且高度是宽度的 1.5 倍 (或接近)
|
||||||
|
if (test_frame.type() == CV_8UC1)
|
||||||
|
{
|
||||||
|
// 修正 frame_height (去除 UV 部分的高度)
|
||||||
|
frame_height_ = (test_frame.rows * 2) / 3;
|
||||||
|
frame_width_ = test_frame.cols;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
frame_width_ = test_frame.cols;
|
||||||
|
frame_height_ = test_frame.rows;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::lock_guard<std::mutex> lock(frame_mutex_);
|
||||||
|
latest_frame_ = test_frame;
|
||||||
|
new_frame_available_ = true;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
printf("RTSP stream opened! Real Res: %dx%d @ %.2f FPS (Mode: %s)\n",
|
||||||
|
frame_width_, frame_height_, frame_fps_,
|
||||||
|
(latest_frame_.type() == CV_8UC1 ? "NV12" : "BGR"));
|
||||||
|
|
||||||
|
// --- 输出部分保持不变 ---
|
||||||
|
std::string gst_out_pipeline =
|
||||||
|
"appsrc name=mysource is-live=true format=3 ! "
|
||||||
|
"queue max-size-buffers=2 leaky=downstream ! "
|
||||||
|
"video/x-raw,format=NV12,width=" +
|
||||||
|
std::to_string(frame_width_) +
|
||||||
|
",height=" + std::to_string(frame_height_) +
|
||||||
|
",framerate=" + std::to_string((int)frame_fps_) + "/1 ! "
|
||||||
|
"mpph264enc gop=25 rc-mode=fixqp qp-init=26 ! "
|
||||||
|
"h264parse ! "
|
||||||
|
"rtspclientsink location=" +
|
||||||
|
output_rtsp_url_ + " latency=0 protocols=tcp";
|
||||||
|
|
||||||
|
GError *error = nullptr;
|
||||||
|
gst_pipeline_ = gst_parse_launch(gst_out_pipeline.c_str(), &error);
|
||||||
|
|
||||||
|
if (error)
|
||||||
|
{
|
||||||
|
spdlog::error("Failed to parse output pipeline: {}", error->message);
|
||||||
|
g_error_free(error);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
gst_appsrc_ = gst_bin_get_by_name(GST_BIN(gst_pipeline_), "mysource");
|
||||||
|
if (!gst_appsrc_)
|
||||||
|
{
|
||||||
|
spdlog::error("Failed to get 'mysource' from pipeline");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
gst_element_set_state(gst_pipeline_, GST_STATE_PLAYING);
|
||||||
|
printf("GStreamer Output Pipeline started manually.\n");
|
||||||
|
|
||||||
|
running_ = true;
|
||||||
|
reading_thread_ = std::thread(&VideoService::reading_loop, this);
|
||||||
|
processing_thread_ = std::thread(&VideoService::processing_loop, this);
|
||||||
|
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
void VideoService::processing_loop() {
|
void VideoService::stop()
|
||||||
cv::Mat frame;
|
{
|
||||||
|
printf("Stopping VideoService...\n");
|
||||||
|
running_ = false;
|
||||||
|
frame_cv_.notify_all();
|
||||||
|
|
||||||
while (running_) {
|
if (reading_thread_.joinable())
|
||||||
{
|
reading_thread_.join();
|
||||||
// 1. (不变) 获取帧
|
if (processing_thread_.joinable())
|
||||||
std::unique_lock<std::mutex> lock(frame_mutex_);
|
processing_thread_.join();
|
||||||
|
|
||||||
frame_cv_.wait(lock, [&] { return new_frame_available_ || !running_; });
|
if (capture_.isOpened())
|
||||||
|
capture_.release();
|
||||||
|
|
||||||
if (!running_) {
|
if (gst_pipeline_)
|
||||||
break;
|
{
|
||||||
}
|
gst_element_set_state(gst_pipeline_, GST_STATE_NULL);
|
||||||
|
gst_object_unref(gst_pipeline_);
|
||||||
|
gst_pipeline_ = nullptr;
|
||||||
|
}
|
||||||
|
if (gst_appsrc_)
|
||||||
|
{
|
||||||
|
gst_object_unref(gst_appsrc_);
|
||||||
|
gst_appsrc_ = nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
frame = latest_frame_.clone();
|
module_->stop();
|
||||||
new_frame_available_ = false;
|
module_.reset();
|
||||||
}
|
printf("VideoService stopped.\n");
|
||||||
|
}
|
||||||
|
|
||||||
if (frame.empty()) {
|
void VideoService::reading_loop()
|
||||||
continue;
|
{
|
||||||
}
|
cv::Mat frame;
|
||||||
if (!module_->process(frame)) {
|
while (running_)
|
||||||
// 模块报告处理失败
|
{
|
||||||
spdlog::warn("{} Module failed to process frame. Skipping.", log_prefix_);
|
if (!capture_.read(frame))
|
||||||
}
|
{
|
||||||
if (writer_.isOpened()) {
|
running_ = false;
|
||||||
writer_.write(frame);
|
break;
|
||||||
}
|
}
|
||||||
}
|
if (frame.empty())
|
||||||
|
continue;
|
||||||
|
{
|
||||||
|
std::lock_guard<std::mutex> lock(frame_mutex_);
|
||||||
|
latest_frame_ = frame;
|
||||||
|
new_frame_available_ = true;
|
||||||
|
}
|
||||||
|
frame_cv_.notify_one();
|
||||||
|
}
|
||||||
|
frame_cv_.notify_all();
|
||||||
|
}
|
||||||
|
|
||||||
spdlog::info("VideoService: Processing loop finished.");
|
void VideoService::processing_loop()
|
||||||
|
{
|
||||||
|
cv::Mat raw_frame;
|
||||||
|
|
||||||
|
// RGA 专用 4K 对齐内存 (用于传给 AI 模块)
|
||||||
|
// 依然保留，因为 module_->process 可能需要稳定的 BGR/RGBA 输入
|
||||||
|
cv::Mat frame_rgba = create_aligned_mat(frame_width_, frame_height_, CV_8UC4);
|
||||||
|
|
||||||
|
// 临时 BGR 帧 (CPU 转换用)
|
||||||
|
cv::Mat frame_bgr;
|
||||||
|
|
||||||
|
if (frame_rgba.empty() || frame_rgba.data == nullptr)
|
||||||
|
{
|
||||||
|
spdlog::error("Fatal: Failed to allocate aligned buffer for RGA!");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<uint8_t> nv12_buffer;
|
||||||
|
|
||||||
|
spdlog::info("Processing thread ready. (CPU NV12->BGR enabled)");
|
||||||
|
|
||||||
|
while (running_)
|
||||||
|
{
|
||||||
|
{
|
||||||
|
std::unique_lock<std::mutex> lock(frame_mutex_);
|
||||||
|
frame_cv_.wait(lock, [&]
|
||||||
|
{ return new_frame_available_ || !running_; });
|
||||||
|
|
||||||
|
if (!running_)
|
||||||
|
break;
|
||||||
|
|
||||||
|
raw_frame = latest_frame_.clone();
|
||||||
|
new_frame_available_ = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (raw_frame.empty())
|
||||||
|
continue;
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------
|
||||||
|
// [关键修正] CPU 格式转换 (NV12 -> BGR)
|
||||||
|
// ---------------------------------------------------------------
|
||||||
|
if (raw_frame.type() == CV_8UC1 && raw_frame.rows == frame_height_ * 3 / 2)
|
||||||
|
{
|
||||||
|
// 输入是 NV12，使用 CPU 转换为 BGR
|
||||||
|
// 这避免了使用不稳定的 GStreamer RGA 插件
|
||||||
|
cv::cvtColor(raw_frame, frame_bgr, cv::COLOR_YUV2BGR_NV12);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// 如果已经是 BGR (fallback)
|
||||||
|
frame_bgr = raw_frame;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------
|
||||||
|
// [准备 AI 输入] BGR -> RGBA (写入对齐内存)
|
||||||
|
// ---------------------------------------------------------------
|
||||||
|
// rkYolov8 内部已经加锁，这里传递 4K 对齐内存也是安全的
|
||||||
|
cv::cvtColor(frame_bgr, frame_rgba, cv::COLOR_BGR2RGBA);
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------
|
||||||
|
// [调用 AI 模块]
|
||||||
|
// ---------------------------------------------------------------
|
||||||
|
// 模块会在 frame_rgba 上进行检测，并在 frame_rgba (或者我们需要传 BGR?)
|
||||||
|
// 等等，module_->process 接收的是引用并绘制结果。
|
||||||
|
// HumanDetectionModule 的 draw_results 是用 opencv 绘图的。
|
||||||
|
// 为了让绘图结果能推流出去，我们应该让 module 处理 frame_bgr。
|
||||||
|
|
||||||
|
// 修正：rkYolov8::infer 内部只读，不修改。
|
||||||
|
// HumanDetectionModule::process 会调用 draw_results 修改图像。
|
||||||
|
// 我们传入 frame_bgr 给 AI 模块 (它内部会转 RGBA 传给 NPU，这没问题)。
|
||||||
|
// 这里的 frame_bgr 是 CPU 内存，不是 4K 对齐的，但 rkYolov8 现在有锁且能处理非对齐。
|
||||||
|
// 或者，为了极致性能和匹配之前的逻辑，我们还是传 frame_rgba 进去？
|
||||||
|
// HumanDetectionModule::process 接收 Mat&。
|
||||||
|
// 如果我们传 frame_rgba，它画框也是画在 RGBA 上。
|
||||||
|
|
||||||
|
if (!module_->process(frame_rgba))
|
||||||
|
{
|
||||||
|
// process fail
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------
|
||||||
|
// [推流部分] RGBA -> NV12
|
||||||
|
// ---------------------------------------------------------------
|
||||||
|
if (gst_appsrc_)
|
||||||
|
{
|
||||||
|
// 将画好框的 RGBA 转回 NV12 推流
|
||||||
|
// 注意：bgr_to_nv12 原本是 BGR->NV12。我们需要适配一下，或者转回 BGR。
|
||||||
|
// 简单的做法：RGBA -> BGR -> NV12 (虽然多了一步，但逻辑简单)
|
||||||
|
cv::Mat temp_bgr;
|
||||||
|
cv::cvtColor(frame_rgba, temp_bgr, cv::COLOR_RGBA2BGR);
|
||||||
|
bgr_to_nv12(temp_bgr, nv12_buffer);
|
||||||
|
|
||||||
|
guint size = nv12_buffer.size();
|
||||||
|
GstBuffer *buffer = gst_buffer_new_allocate(NULL, size, NULL);
|
||||||
|
GstMapInfo map;
|
||||||
|
gst_buffer_map(buffer, &map, GST_MAP_WRITE);
|
||||||
|
memcpy(map.data, nv12_buffer.data(), size);
|
||||||
|
gst_buffer_unmap(buffer, &map);
|
||||||
|
|
||||||
|
GstFlowReturn ret;
|
||||||
|
g_signal_emit_by_name(gst_appsrc_, "push-buffer", buffer, &ret);
|
||||||
|
gst_buffer_unref(buffer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 释放手动分配的内存
|
||||||
|
if (frame_rgba.data)
|
||||||
|
free(frame_rgba.data);
|
||||||
|
|
||||||
|
spdlog::info("VideoService: Processing loop finished.");
|
||||||
}
|
}
|
||||||
|
|
@ -12,44 +12,51 @@
|
||||||
#include <opencv2/videoio.hpp>
|
#include <opencv2/videoio.hpp>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <thread>
|
#include <thread>
|
||||||
|
#include <gst/gst.h>
|
||||||
|
#include <gst/app/gstappsrc.h>
|
||||||
|
|
||||||
class VideoService {
|
class VideoService
|
||||||
|
{
|
||||||
public:
|
public:
|
||||||
/**
|
/**
|
||||||
* @brief 构造函数变更：
|
* @brief 构造函数变更：
|
||||||
* 不再接收 model_path 和 thread_num，
|
* 不再接收 model_path 和 thread_num，
|
||||||
* 而是通过依赖注入接收一个抽象的 AI 模块。
|
* 而是通过依赖注入接收一个抽象的 AI 模块。
|
||||||
*/
|
*/
|
||||||
VideoService(std::unique_ptr<IAnalysisModule> module, std::string input_url,
|
VideoService(std::unique_ptr<IAnalysisModule> module, std::string input_url,
|
||||||
std::string output_rtsp_url, nlohmann::json module_config);
|
std::string output_rtsp_url, nlohmann::json module_config);
|
||||||
|
|
||||||
~VideoService();
|
~VideoService();
|
||||||
|
|
||||||
bool start();
|
bool start();
|
||||||
void stop();
|
void stop();
|
||||||
|
|
||||||
private:
|
private:
|
||||||
void processing_loop();
|
void processing_loop();
|
||||||
void reading_loop();
|
void reading_loop();
|
||||||
|
void bgr_to_nv12(const cv::Mat &src, std::vector<uint8_t> &dst);
|
||||||
|
cv::Mat create_aligned_mat(int width, int height, int type);
|
||||||
|
|
||||||
std::unique_ptr<IAnalysisModule> module_;
|
std::unique_ptr<IAnalysisModule> module_;
|
||||||
nlohmann::json module_config_;
|
nlohmann::json module_config_;
|
||||||
std::string input_url_;
|
std::string input_url_;
|
||||||
std::string output_rtsp_url_;
|
std::string output_rtsp_url_;
|
||||||
|
|
||||||
int frame_width_ = 0;
|
int frame_width_ = 0;
|
||||||
int frame_height_ = 0;
|
int frame_height_ = 0;
|
||||||
double frame_fps_ = 0.0;
|
double frame_fps_ = 0.0;
|
||||||
cv::VideoCapture capture_;
|
cv::VideoCapture capture_;
|
||||||
cv::VideoWriter writer_;
|
cv::VideoWriter writer_;
|
||||||
|
GstElement *gst_pipeline_ = nullptr;
|
||||||
|
GstElement *gst_appsrc_ = nullptr;
|
||||||
|
|
||||||
std::thread processing_thread_;
|
std::thread processing_thread_;
|
||||||
std::thread reading_thread_;
|
std::thread reading_thread_;
|
||||||
std::atomic<bool> running_{false};
|
std::atomic<bool> running_{false};
|
||||||
std::mutex frame_mutex_;
|
std::mutex frame_mutex_;
|
||||||
std::condition_variable frame_cv_;
|
std::condition_variable frame_cv_;
|
||||||
cv::Mat latest_frame_;
|
cv::Mat latest_frame_;
|
||||||
bool new_frame_available_{false};
|
bool new_frame_available_{false};
|
||||||
|
|
||||||
std::string log_prefix_;
|
std::string log_prefix_;
|
||||||
};
|
};
|
||||||
Loading…
Reference in New Issue