new model

README.md

@@ -2,7 +2,7 @@
  * @Author: yangxue && xue.yang@waytous.com
  * @Date: 2025-04-07 09:26:17
  * @LastEditors: yangxue xue.yang@waytous.com
- * @LastEditTime: 2025-04-10 09:45:19
+ * @LastEditTime: 2025-04-17 06:22:11
  * @FilePath: /ubuntu/projects/ros_camera_bev/README.md
  * @Description: 
  * 
@@ -21,7 +21,7 @@ catkin_make -DCATKIN_WHITELIST_PACKAGES="perception_camera_bev" -DCUDA_nppicom_L
 
 
 # 参数修改
-    ■ 有*.engine文件则删除（不要删除onnx文件）：/configs/weights/ips_241107m_stone_best_fp16.engine
+    ■ 如果部署到不同平台上，有*.engine文件则删除，程序重新生成（不要删除onnx文件）：/configs/weights/ips_241107m_stone_best_fp16.engine
     ■ 修改参数文件 configs/CameraBevParam.yaml
       ● rgb_sub_topic_name 图像数据话题
       ● pointcloud_sub_topic_name 点云数据话题
@@ -37,6 +37,7 @@ catkin_make -DCATKIN_WHITELIST_PACKAGES="perception_camera_bev" -DCUDA_nppicom_L
 ```
 bash launch.sh
 ```
+- 注意：部署到新平台上，第一次运行会生成engine文件，耗时较长是正常现象。
 
 # 本地天准域控测试的一些关键命令
 ```
@@ -58,5 +59,5 @@ rviz
 ```
 # 压缩部署关键文件
 ```
-zip -r ros_camera_bev.zip ros_camera_bev/ -x "ros_camera_bev/.vscode/*" -x "ros_camera_bev/devel/*" -x "ros_camera_bev/build/*" -x "ros_camera_bev/sdk/*" -x "ros_camera_bev/src/camera_back_warn/*" -x "*.engine" -x "*.bag" -x "ros_camera_bev/.git/*" -x "*.gitignore"
+zip -r ros_camera_bev.zip ros_camera_bev/ -x "ros_camera_bev/.vscode/*" -x "ros_camera_bev/devel/*" -x "ros_camera_bev/build/*" -x "ros_camera_bev/sdk/*" -x "ros_camera_bev/src/camera_back_warn/*" -x "*.engine" -x "*.bag" -x "ros_camera_bev/.git/*" -x "*.gitignore" -x "*.catkin_workspace"
 ```
\ No newline at end of file

src/camera_bev_infer/configs/CameraBevParam.yaml

@@ -12,7 +12,7 @@ device: 0
 period_time: 5
 vis_res: true
 vis_frame_id: livox_frame_hap # vehicle_link
-vis_project: true
+vis_project: false
 # ["pedestrian", "vehicle", "stone"]
 pub_classes: ["pedestrian", "vehicle"] # ["stone", "cone"]
 obstacle_height_threshold: 25 # cm

src/camera_bev_infer/configs/weights/multi_yolov5m.yaml

@@ -28,8 +28,8 @@ units:
         inputNames: [resizeNormImages]
         outputNames: [detections, seg_protos, depths, semantics]
         runMode: 1 # 0-fp32 1-fp16 2-int8 (int8 not supported)
-        weightsPath: "configs/weights/ips_241107m_stone_best.onnx" # "configs/weights/isp_x0228m_last.onnx"
-        engineFile: "configs/weights/ips_241107m_stone_best_fp16.engine" #  "configs/weights/isp_x0228m_last_fp16.engine"
+        weightsPath: "configs/weights/ips_2504m_xisan_ft.onnx" # "configs/weights/isp_x0228m_last.onnx"
+        engineFile: "configs/weights/ips_2504m_xisan_ft_fp16.engine" #  "configs/weights/isp_x0228m_last_fp16.engine"
         inferDynamic: false
         inferBatchSize: 1
         inputWidth: 960

src/camera_bev_infer/src/sensor_manager.cpp

@@ -5,7 +5,7 @@
  * @Date: 2023-09-03 02:18:07
  * @email: xin.wang@waytous.com
  * @LastEditors: yangxue xue.yang@waytous.com
- * @LastEditTime: 2025-04-17 03:47:34
+ * @LastEditTime: 2025-04-18 01:48:25
  */
 
 
@@ -286,6 +286,7 @@ void sensorManager::publishDetection()
             // int8  label_type = 9
             ob.label_type = (name2type.find(obj.names) == name2type.end() ? Perception_Default_Classification : name2type[obj.names]);
             obTrack.track_type = (name2type.find(obj.names) == name2type.end() ? Perception_Default_Classification : name2type[obj.names]);
+            obTrack.bounding_box.label = obTrack.track_type;
 
             //float32[]  type_probs = 10
             // std::vector<float> probs(waytous_perception_msgs::Object::TYPE_UNKNOWN_STATIC + 1, 0); // main-type length

src/camera_bev_infer/src/vision_transform/vision_3d.cpp

@@ -5,7 +5,7 @@
  * @Date: 2023-09-03 03:13:41
  * @email: xin.wang@waytous.com
  * @LastEditors: yangxue xue.yang@waytous.com
- * @LastEditTime: 2025-04-10 09:24:17
+ * @LastEditTime: 2025-04-21 08:11:54
  */
 
 
@@ -922,6 +922,9 @@ int Visione_3D::Process_Instance_Objects(object_Seg_info& obj, const cv::Mat& in
     int objY1 = objBin.rows, objY2 = 0;
     int objX1 = objBin.cols, objX2 = 0;
 
+    int nearest_px = 100;
+    int nearest_py = 100;
+
     //目标 mask
     // clock_t start_ = clock();
     int mask_count = 0, project_count = 0;
@@ -944,7 +947,8 @@ int Visione_3D::Process_Instance_Objects(object_Seg_info& obj, const cv::Mat& in
             project_count++;
             
             auto& p3d = pcl_cloud->points[index];
-            ix = int(p3d.y/BEV_GRID_RESOLUTION + BEV_GridMap::m_nCenterX);
+            // BEV Frame: left-bottom as origin, ix: right， iy: forward
+            ix = int(p3d.y/BEV_GRID_RESOLUTION + BEV_GridMap::m_nCenterX); // lidar frame -> BEV frame
             iy = int(p3d.x/BEV_GRID_RESOLUTION + BEV_GridMap::m_nCenterY);
             if( ix >= BEV_IAGE_WIDTH || ix < 0 || 
                 iy >= BEV_IAGE_HEIGHT || iy < 0 )
@@ -953,12 +957,19 @@ int Visione_3D::Process_Instance_Objects(object_Seg_info& obj, const cv::Mat& in
             }
             
             if( ix !=0 && iy != 0)
-            {
+            {   
+                // 当前目标所有点云的bev范围
                 objY1 = MIN(iy, objY1);
                 objY2 = MAX(iy, objY2);    
                 objX1 = MIN(ix, objX1);
                 objX2 = MAX(ix, objX2);
 
+                // 找最近点
+                if(nearest_py > iy) {
+                    nearest_py = iy;
+                    nearest_px = ix;
+                }
+
                 objBin.at<uchar>(iy, ix) = 255;
                 objBinCount.at<uchar>(iy, ix)  = MIN(255, objBinCount.at<uchar>(iy, ix)+1);
                 cv::Point3d p3d_(ix, iy, p3d.z);
@@ -988,8 +999,15 @@ int Visione_3D::Process_Instance_Objects(object_Seg_info& obj, const cv::Mat& in
     //目标bev矩形区域
     InstanceBevObject insobj;
     if(nValidNum >= 3 && objRc.width > 0 && objRc.height > 0)
-    {
+    {   
+        // 根据目标bev点云，计算其bev边界框
         insobj.rrc = getObjectAreaRect(objBin, objBinCount, objRc, insobj.vPoints, color, obj.name);
+        if(obj.name == "pedestrian") {
+            std::cout << "update pedestrian position: " << nearest_py << " --> " << insobj.rrc.center.y << std::endl;
+            insobj.rrc.center.x = nearest_px;
+            insobj.rrc.center.y = nearest_py;
+        }
+
         if(insobj.rrc.size.height > 0 && insobj.rrc.size.width > 0)
         {
             pointEnough = true;
@@ -1173,9 +1191,6 @@ int Visione_3D::Process_objects(const cv::Mat& image, const cv::Mat& depth, Clou
             }
         }
     }
-    return 0;
-
-
-    
+    return 0;    
 };