Skip to content

Commit e57f4a5

Browse files
amadeuszszmanatopre-commit-ci-lite[bot]
authored andcommitted
feat(autoware_ptv3): source cloud reconstruction & entropy (autowarefoundation#12547)
* feat(autoware_ptv3): source cloud reconstruction & entropy Signed-off-by: Amadeusz Szymko <amadeusz.szymko.2@tier4.jp> * refactor(ptv3): clarify reconstruction feature handling Signed-off-by: Amadeusz Szymko <amadeusz.szymko.2@tier4.jp> * refactor(autoware_ptv3): remove unnecessary buffers clear Signed-off-by: Amadeusz Szymko <amadeusz.szymko.2@tier4.jp> * refactor(ptv3): simplify source reconstruction buffers Signed-off-by: Amadeusz Szymko <amadeusz.szymko.2@tier4.jp> * fix(autoware_ptv3): remove redundant memory clear Signed-off-by: Amadeusz Szymko <amadeusz.szymko.2@tier4.jp> * feat(autoware_ptv3): add __restrict__ Signed-off-by: Amadeusz Szymko <amadeusz.szymko.2@tier4.jp> Co-authored-by: Manato Hirabayashi <3022416+manato@users.noreply.github.com> * style(pre-commit): autofix * feat(autoware_ptv3): add restrict Signed-off-by: Amadeusz Szymko <amadeusz.szymko.2@tier4.jp> * feat(autoware_ptv3): parallelizaiton Signed-off-by: Amadeusz Szymko <amadeusz.szymko.2@tier4.jp> --------- Signed-off-by: Amadeusz Szymko <amadeusz.szymko.2@tier4.jp> Co-authored-by: Manato Hirabayashi <3022416+manato@users.noreply.github.com> Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com>
1 parent 47b4ac9 commit e57f4a5

11 files changed

Lines changed: 351 additions & 33 deletions

File tree

‎perception/autoware_ptv3/config/ptv3.param.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
# network
66
trt_precision: fp16
77
cloud_capacity: 2000000
8+
source_reconstruction: full
89
onnx_path: "$(var model_path)/ptv3.onnx"
910
engine_path: "$(var model_path)/ptv3.engine"
1011
filter:

‎perception/autoware_ptv3/include/autoware/ptv3/postprocess/postprocess_kernel.hpp‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,12 +35,23 @@ class PostprocessCuda
3535

3636
void createVisualizationPointcloud(
3737
const float * input_features, const std::int64_t * pred_labels, float * output_points,
38-
std::size_t num_points);
38+
std::size_t num_classes, std::size_t num_points);
3939

4040
void createSegmentationPointcloud(
4141
const float * input_features, const std::int64_t * pred_labels, const float * pred_probs,
4242
std::uint8_t * output_points, std::size_t num_classes, std::size_t num_points);
4343

44+
void reconstructPartial(
45+
const std::int64_t * inverse_map, const std::int64_t * voxel_labels, const float * voxel_probs,
46+
std::int64_t * output_labels, float * output_probs, std::size_t num_classes,
47+
std::size_t num_cropped_points, std::size_t num_voxels);
48+
49+
void reconstructFull(
50+
const std::uint32_t * crop_mask, const std::uint32_t * crop_indices,
51+
const std::int64_t * inverse_map, const std::int64_t * voxel_labels, const float * voxel_probs,
52+
std::int64_t * output_labels, float * output_probs, std::size_t num_classes,
53+
std::size_t num_points, std::size_t num_voxels);
54+
4455
std::size_t createFilteredPointcloud(
4556
const void * compact_input_points, CloudFormat input_format, CloudFormat output_format,
4657
const float * pred_probs, void * output_points, std::size_t num_classes,

‎perception/autoware_ptv3/include/autoware/ptv3/preprocess/preprocess_kernel.hpp‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,11 @@ class PreprocessCuda
3535
std::size_t generateFeatures(
3636
const void * input_data, CloudFormat input_format, unsigned int num_points,
3737
float * voxel_features, std::int64_t * voxel_coords, std::int64_t * voxel_hashes,
38-
void * compact_points);
38+
void * compact_points, float * reconstruction_features, void * cropped_source_points,
39+
std::int64_t * inverse_map, std::size_t * num_cropped_points);
40+
41+
[[nodiscard]] const std::uint32_t * cropMask() const { return crop_mask_d_.get(); }
42+
[[nodiscard]] const std::uint32_t * cropIndices() const { return crop_indices_d_.get(); }
3943

4044
private:
4145
PTv3Config config_;

‎perception/autoware_ptv3/include/autoware/ptv3/ptv3_config.hpp‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,12 @@
2828
namespace autoware::ptv3
2929
{
3030

31+
enum class SourceReconstruction {
32+
NONE,
33+
PARTIAL,
34+
FULL,
35+
};
36+
3137
class PTv3Config
3238
{
3339
public:
@@ -36,7 +42,8 @@ class PTv3Config
3642
const std::vector<std::int64_t> & voxels_num, const std::vector<float> & point_cloud_range,
3743
const std::vector<float> & voxel_size, const std::vector<std::string> & class_names,
3844
const std::vector<std::int64_t> & palette, const float filter_class_probability_threshold,
39-
const std::vector<std::string> & filter_classes, const std::string & filter_output_format)
45+
const std::vector<std::string> & filter_classes, const std::string & filter_output_format,
46+
const std::string & source_reconstruction)
4047
{
4148
plugins_path_ = plugins_path;
4249

@@ -91,6 +98,21 @@ class PTv3Config
9198
filter_class_probability_threshold_ = filter_class_probability_threshold;
9299
filter_class_indices_ = make_filter_class_indices(class_names_, filter_classes);
93100
filter_output_format_ = filter_output_format;
101+
source_reconstruction_ = parse_source_reconstruction(source_reconstruction);
102+
}
103+
104+
static SourceReconstruction parse_source_reconstruction(const std::string & value)
105+
{
106+
if (value == "none") {
107+
return SourceReconstruction::NONE;
108+
}
109+
if (value == "partial") {
110+
return SourceReconstruction::PARTIAL;
111+
}
112+
if (value == "full") {
113+
return SourceReconstruction::FULL;
114+
}
115+
throw std::runtime_error("source_reconstruction must be one of: 'none', 'partial', or 'full'.");
94116
}
95117

96118
static std::vector<std::uint32_t> make_filter_class_indices(
@@ -155,6 +177,7 @@ class PTv3Config
155177
float filter_class_probability_threshold_{};
156178
std::vector<std::uint32_t> filter_class_indices_;
157179
std::string filter_output_format_;
180+
SourceReconstruction source_reconstruction_{SourceReconstruction::NONE};
158181

159182
// Common network parameters
160183
std::int64_t cloud_capacity_{};

‎perception/autoware_ptv3/include/autoware/ptv3/ptv3_trt.hpp‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,8 +100,16 @@ class PTV3_PUBLIC PTv3TRT
100100

101101
// Preprocess outputs
102102
std::int64_t num_voxels_{0};
103+
std::int64_t num_cropped_points_{0}; // only for partial
104+
std::int64_t num_source_points_{0}; // only for full
105+
const void * current_input_data_{nullptr}; // only for full
103106

104107
CudaUniquePtr<std::uint8_t[]> compact_points_d_{nullptr};
108+
CudaUniquePtr<std::uint8_t[]> cropped_source_points_d_{nullptr}; // only for partial
109+
CudaUniquePtr<float[]> reconstructed_features_d_{nullptr}; // only for partial and full
110+
CudaUniquePtr<std::int64_t[]> inverse_map_d_{nullptr}; // only for partial and full
111+
CudaUniquePtr<std::int64_t[]> reconstructed_labels_d_{nullptr}; // only for partial and full
112+
CudaUniquePtr<float[]> reconstructed_probs_d_{nullptr}; // only for partial and full
105113
CudaUniquePtr<std::int64_t[]> grid_coord_d_{nullptr};
106114
CudaUniquePtr<float[]> feat_d_{nullptr};
107115
CudaUniquePtr<std::int64_t[]> serialized_code_d_{nullptr};

‎perception/autoware_ptv3/launch/ptv3.launch.xml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,13 +40,13 @@
4040
<group unless="$(var use_pointcloud_container)">
4141
<node pkg="autoware_ptv3" exec="autoware_ptv3_node" name="ptv3" output="screen" args="--ros-args --log-level $(var log_level)">
4242
<remap from="~/input/pointcloud" to="$(var input/pointcloud)"/>
43-
<remap from="~/input/pointcloud/cuda" to="$(var input/pointcloud)/cuda"/>
43+
<remap from="~/input/pointcloud/cuda" to="$(var input/pointcloud)/cuda/dummy"/>
4444
<remap from="~/output/pointcloud/segmentation" to="$(var output/pointcloud/segmentation)"/>
45-
<remap from="~/output/pointcloud/segmentation/cuda" to="$(var output/pointcloud/segmentation)/cuda"/>
45+
<remap from="~/output/pointcloud/segmentation/cuda" to="$(var output/pointcloud/segmentation)/cuda/dummy"/>
4646
<remap from="~/output/pointcloud/visualization" to="$(var output/pointcloud/visualization)"/>
47-
<remap from="~/output/pointcloud/visualization/cuda" to="$(var output/pointcloud/visualization)/cuda"/>
47+
<remap from="~/output/pointcloud/visualization/cuda" to="$(var output/pointcloud/visualization)/cuda/dummy"/>
4848
<remap from="~/output/pointcloud/filtered" to="$(var output/pointcloud/filtered)"/>
49-
<remap from="~/output/pointcloud/filtered/cuda" to="$(var output/pointcloud/filtered)/cuda"/>
49+
<remap from="~/output/pointcloud/filtered/cuda" to="$(var output/pointcloud/filtered)/cuda/dummy"/>
5050

5151
<param from="$(var model_param_path)" allow_substs="true"/>
5252
<param from="$(var ml_package_param_path)" allow_substs="true"/>

‎perception/autoware_ptv3/lib/postprocess/postprocess_kernel.cu‎

Lines changed: 106 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -26,19 +26,21 @@ struct OutputSegmentationPointType
2626
float z;
2727
std::uint8_t class_id;
2828
float probability;
29+
float entropy;
2930
} __attribute__((packed));
3031

3132
__global__ void createVisualizationPointcloudKernel(
3233
const float4 * input_features, const float * colors, const std::int64_t * labels,
33-
float4 * output_points, std::size_t num_points)
34+
float4 * output_points, std::size_t num_classes, std::size_t num_points)
3435
{
3536
const auto idx = static_cast<std::uint32_t>(blockIdx.x * blockDim.x + threadIdx.x);
3637
if (idx >= num_points) {
3738
return;
3839
}
3940

4041
const auto label = labels[idx];
41-
const auto color = colors[label];
42+
const auto color =
43+
label >= 0 && static_cast<std::size_t>(label) < num_classes ? colors[label] : 0.0f;
4244

4345
output_points[idx] =
4446
make_float4(input_features[idx].x, input_features[idx].y, input_features[idx].z, color);
@@ -55,11 +57,78 @@ __global__ void createSegmentationPointcloudKernel(
5557

5658
const auto input_point = input_features[idx];
5759
const auto label = labels[idx];
60+
const bool has_valid_label = label >= 0 && static_cast<std::size_t>(label) < num_classes;
61+
float entropy = 0.0f;
62+
for (std::size_t class_idx = 0; class_idx < num_classes; ++class_idx) {
63+
const auto probability = pred_probs[idx * num_classes + class_idx];
64+
if (probability > 0.0f) {
65+
entropy -= probability * logf(probability);
66+
}
67+
}
68+
if (num_classes > 1) {
69+
entropy /= logf(static_cast<float>(num_classes));
70+
}
5871
output_points[idx].x = input_point.x;
5972
output_points[idx].y = input_point.y;
6073
output_points[idx].z = input_point.z;
61-
output_points[idx].class_id = static_cast<std::uint8_t>(label);
62-
output_points[idx].probability = pred_probs[idx * num_classes + label];
74+
output_points[idx].class_id = has_valid_label ? static_cast<std::uint8_t>(label) : 255U;
75+
output_points[idx].probability = has_valid_label ? pred_probs[idx * num_classes + label] : 0.0f;
76+
output_points[idx].entropy = entropy;
77+
}
78+
79+
__global__ void reconstructPartialKernel(
80+
const std::int64_t * __restrict__ inverse_map, const std::int64_t * __restrict__ voxel_labels,
81+
const float * __restrict__ voxel_probs, std::int64_t * __restrict__ output_labels,
82+
float * __restrict__ output_probs, std::size_t num_classes, std::size_t num_cropped_points,
83+
std::size_t num_voxels)
84+
{
85+
const auto point_idx = blockIdx.x * blockDim.y + threadIdx.y;
86+
const auto class_idx = blockIdx.y * blockDim.x + threadIdx.x;
87+
88+
if (point_idx >= num_cropped_points || class_idx >= num_classes) return;
89+
90+
const auto voxel_idx = inverse_map[point_idx];
91+
const bool has_valid_voxel = voxel_idx >= 0 && static_cast<std::size_t>(voxel_idx) < num_voxels;
92+
if (class_idx == 0) {
93+
output_labels[point_idx] = has_valid_voxel ? voxel_labels[voxel_idx] : 255;
94+
}
95+
96+
output_probs[point_idx * num_classes + class_idx] =
97+
has_valid_voxel ? voxel_probs[static_cast<std::size_t>(voxel_idx) * num_classes + class_idx]
98+
: 0.0f;
99+
}
100+
101+
__global__ void reconstructFullKernel(
102+
const std::uint32_t * __restrict__ crop_mask, const std::uint32_t * __restrict__ crop_indices,
103+
const std::int64_t * __restrict__ inverse_map, const std::int64_t * __restrict__ voxel_labels,
104+
const float * __restrict__ voxel_probs, std::int64_t * __restrict__ output_labels,
105+
float * __restrict__ output_probs, std::size_t num_classes, std::size_t num_points,
106+
std::size_t num_voxels)
107+
{
108+
const auto point_idx = blockIdx.x * blockDim.y + threadIdx.y;
109+
const auto class_idx = blockIdx.y * blockDim.x + threadIdx.x;
110+
111+
if (point_idx >= num_points || class_idx >= num_classes) return;
112+
113+
const auto mask = crop_mask[point_idx];
114+
if (mask == 0) {
115+
if (class_idx == 0) output_labels[point_idx] = 255;
116+
return;
117+
}
118+
119+
const auto cropped_idx = crop_indices[point_idx] - 1;
120+
const auto voxel_idx = inverse_map[cropped_idx];
121+
if (voxel_idx < 0 || static_cast<std::size_t>(voxel_idx) >= num_voxels) {
122+
if (class_idx == 0) output_labels[point_idx] = 255;
123+
return;
124+
}
125+
126+
if (class_idx == 0) {
127+
output_labels[point_idx] = voxel_labels[voxel_idx];
128+
}
129+
130+
output_probs[point_idx * num_classes + class_idx] =
131+
voxel_probs[static_cast<std::size_t>(voxel_idx) * num_classes + class_idx];
63132
}
64133

65134
template <typename OutputPointT>
@@ -213,13 +282,13 @@ PostprocessCuda::PostprocessCuda(const PTv3Config & config, cudaStream_t stream)
213282

214283
void PostprocessCuda::createVisualizationPointcloud(
215284
const float * input_features, const std::int64_t * labels, float * output_points,
216-
std::size_t num_points)
285+
std::size_t num_classes, std::size_t num_points)
217286
{
218287
auto num_blocks = divup(num_points, config_.threads_per_block_);
219288

220289
createVisualizationPointcloudKernel<<<num_blocks, config_.threads_per_block_, 0, stream_>>>(
221290
reinterpret_cast<const float4 *>(input_features), color_map_d_.get(), labels,
222-
reinterpret_cast<float4 *>(output_points), num_points);
291+
reinterpret_cast<float4 *>(output_points), num_classes, num_points);
223292

224293
CHECK_CUDA_ERROR(cudaStreamSynchronize(stream_));
225294
}
@@ -237,6 +306,37 @@ void PostprocessCuda::createSegmentationPointcloud(
237306
CHECK_CUDA_ERROR(cudaStreamSynchronize(stream_));
238307
}
239308

309+
void PostprocessCuda::reconstructPartial(
310+
const std::int64_t * inverse_map, const std::int64_t * voxel_labels, const float * voxel_probs,
311+
std::int64_t * output_labels, float * output_probs, std::size_t num_classes,
312+
std::size_t num_cropped_points, std::size_t num_voxels)
313+
{
314+
auto block = dim3(32, 8);
315+
auto grid = dim3(divup(num_cropped_points, block.y), divup(num_classes, block.x));
316+
317+
reconstructPartialKernel<<<grid, block, 0, stream_>>>(
318+
inverse_map, voxel_labels, voxel_probs, output_labels, output_probs, num_classes,
319+
num_cropped_points, num_voxels);
320+
321+
CHECK_CUDA_ERROR(cudaStreamSynchronize(stream_));
322+
}
323+
324+
void PostprocessCuda::reconstructFull(
325+
const std::uint32_t * crop_mask, const std::uint32_t * crop_indices,
326+
const std::int64_t * inverse_map, const std::int64_t * voxel_labels, const float * voxel_probs,
327+
std::int64_t * output_labels, float * output_probs, std::size_t num_classes,
328+
std::size_t num_points, std::size_t num_voxels)
329+
{
330+
auto block = dim3(32, 8);
331+
auto grid = dim3(divup(num_points, block.y), divup(num_classes, block.x));
332+
333+
reconstructFullKernel<<<grid, block, 0, stream_>>>(
334+
crop_mask, crop_indices, inverse_map, voxel_labels, voxel_probs, output_labels, output_probs,
335+
num_classes, num_points, num_voxels);
336+
337+
CHECK_CUDA_ERROR(cudaStreamSynchronize(stream_));
338+
}
339+
240340
std::size_t PostprocessCuda::createFilteredPointcloud(
241341
const void * compact_input_points, CloudFormat input_format, CloudFormat output_format,
242342
const float * pred_probs, void * output_points, std::size_t num_classes, std::size_t num_points)

0 commit comments

Comments
 (0)