Open3D (C++ API)  0.17.0
ContinuousConvTransposeBackpropFilter.h
Go to the documentation of this file.
1// ----------------------------------------------------------------------------
2// - Open3D: www.open3d.org -
3// ----------------------------------------------------------------------------
4// Copyright (c) 2018-2023 www.open3d.org
5// SPDX-License-Identifier: MIT
6// ----------------------------------------------------------------------------
7
8#pragma once
9
10#include <tbb/parallel_for.h>
11
12#include <mutex>
13
15
16namespace open3d {
17namespace ml {
18namespace impl {
19
22template <class TFeat,
23 class TOut,
24 class TReal,
25 class TIndex,
26 InterpolationMode INTERPOLATION,
27 CoordinateMapping MAPPING,
28 bool ALIGN_CORNERS,
29 bool INDIVIDUAL_EXTENT,
30 bool ISOTROPIC_EXTENT,
31 bool NORMALIZE>
32void _CConvTransposeBackpropFilterCPU(TOut* filter_backprop,
33 const std::vector<int>& filter_dims,
34 size_t num_out,
35 const TReal* out_positions,
36 const TFeat* out_importance,
37 size_t num_inp,
38 const TReal* inp_positions,
39 const TFeat* inp_features,
40 const TFeat* inp_neighbors_importance_sum,
41 const int64_t* inp_neighbors_row_splits,
42 size_t neighbors_index_size,
43 const TIndex* neighbors_index,
44 const TFeat* neighbors_importance,
45 const int64_t* neighbors_row_splits,
46 const TReal* extents,
47 const TReal* offsets,
48 const TFeat* out_features_gradient) {
49 const bool NEIGHBOR_IMPORTANCE = neighbors_importance;
50 const int VECSIZE = 32;
51 typedef Eigen::Array<TReal, VECSIZE, 1> Vec_t;
52 typedef InterpolationVec<TReal, VECSIZE, INTERPOLATION> InterpolationVec_t;
53 InterpolationVec_t interpolation;
54
55 const int in_channels = filter_dims[filter_dims.size() - 2];
56 const int out_channels = filter_dims[filter_dims.size() - 1];
57
58 int spatial_filter_size = 1;
59 for (int i = 0; i < 3; ++i) spatial_filter_size *= filter_dims[i];
60 Eigen::Array<int, 3, 1> filter_size_xyz(filter_dims[2], filter_dims[1],
61 filter_dims[0]);
62
63 memset(filter_backprop, 0,
64 sizeof(TOut) * spatial_filter_size * in_channels * out_channels);
65 std::mutex filter_backprop_mutex;
66
67 tbb::parallel_for(
68 tbb::blocked_range<size_t>(0, num_out, 32),
69 [&](const tbb::blocked_range<size_t>& r) {
70 int range_length = r.end() - r.begin();
71
72 Eigen::Matrix<TFeat, Eigen::Dynamic, Eigen::Dynamic> B(
73 in_channels * spatial_filter_size, range_length);
74 B.setZero();
75 Eigen::Matrix<TFeat, Eigen::Dynamic, Eigen::Dynamic> C(
76 out_channels, range_length);
77
78 typedef Eigen::Array<TFeat, VECSIZE, Eigen::Dynamic> Matrix;
79 Matrix infeat(VECSIZE, in_channels);
80
81 Eigen::Array<TReal, 3, 1> offsets_(offsets[0], offsets[1],
82 offsets[2]);
83
84 Eigen::Array<TReal, VECSIZE, 3> inv_extents;
85 if (INDIVIDUAL_EXTENT == false) {
86 if (ISOTROPIC_EXTENT) {
87 inv_extents = 1 / extents[0];
88 } else {
89 inv_extents.col(0) = 1 / extents[0];
90 inv_extents.col(1) = 1 / extents[1];
91 inv_extents.col(2) = 1 / extents[2];
92 }
93 }
94
95 for (size_t out_idx = r.begin(); out_idx != r.end();
96 ++out_idx) {
97 const int out_col = out_idx - r.begin();
98 const size_t neighbor_start = neighbors_row_splits[out_idx];
99 const size_t neighbor_end =
100 neighbors_row_splits[out_idx + 1];
101
102 C.col(out_col) = Eigen::Map<
103 const Eigen::Array<TFeat, Eigen::Dynamic, 1>>(
104 out_features_gradient + out_idx * out_channels,
105 out_channels, 1);
106
107 typename InterpolationVec_t::Weight_t interp_weights;
108 typename InterpolationVec_t::Idx_t interp_indices;
109
110 int vec_valid_count = 0;
111 Vec_t x, y, z;
112
113 // set to zero to avoid problems with vectors with less than
114 // VECSIZE valid entries
115 x.setZero();
116 y.setZero();
117 z.setZero();
118 for (size_t n = neighbor_start; n < neighbor_end; ++n) {
119 const size_t inp_idx = neighbors_index[n];
120
121 const int i = vec_valid_count;
122 x(i) = out_positions[out_idx * 3 + 0] -
123 inp_positions[inp_idx * 3 + 0];
124 y(i) = out_positions[out_idx * 3 + 1] -
125 inp_positions[inp_idx * 3 + 1];
126 z(i) = out_positions[out_idx * 3 + 2] -
127 inp_positions[inp_idx * 3 + 2];
128
129 if (INDIVIDUAL_EXTENT) {
130 if (ISOTROPIC_EXTENT) {
131 inv_extents.row(i) = 1 / extents[inp_idx];
132 } else {
133 inv_extents(i, 0) =
134 1 / extents[3 * inp_idx + 0];
135 inv_extents(i, 1) =
136 1 / extents[3 * inp_idx + 1];
137 inv_extents(i, 2) =
138 1 / extents[3 * inp_idx + 2];
139 }
140 }
141
142 TFeat n_importance = NEIGHBOR_IMPORTANCE
143 ? neighbors_importance[n]
144 : TFeat(1);
145 for (int ic = 0; ic < in_channels; ++ic)
146 infeat(i, ic) =
147 inp_features[inp_idx * in_channels + ic] *
148 n_importance;
149
150 if (NORMALIZE) {
151 TFeat normalizer(1);
152 if (NEIGHBOR_IMPORTANCE) {
153 if (inp_neighbors_importance_sum[inp_idx] !=
154 TFeat(0))
155 normalizer /= inp_neighbors_importance_sum
156 [inp_idx];
157 } else {
158 size_t num_inp_neighbors;
159 const size_t inp_neighbor_start =
160 inp_neighbors_row_splits[inp_idx];
161 const size_t inp_neighbor_end =
162 inp_neighbors_row_splits[inp_idx + 1];
163 num_inp_neighbors =
164 inp_neighbor_end - inp_neighbor_start;
165 if (num_inp_neighbors > 0)
166 normalizer /= TFeat(num_inp_neighbors);
167 }
168 for (int ic = 0; ic < in_channels; ++ic)
169 infeat(i, ic) *= normalizer;
170 }
171
172 ++vec_valid_count;
173 if (vec_valid_count == VECSIZE ||
174 n + 1 == neighbor_end) {
175 ComputeFilterCoordinates<ALIGN_CORNERS, MAPPING>(
176 x, y, z, filter_size_xyz, inv_extents,
177 offsets_);
178 interpolation.Interpolate(
179 interp_weights, interp_indices, x, y, z,
180 filter_size_xyz, in_channels);
181 for (int k = 0; k < vec_valid_count; ++k) {
182 for (int j = 0; j < InterpolationVec_t::Size();
183 ++j) {
184 for (int ic = 0; ic < in_channels; ++ic)
185 B(interp_indices(j, k) + ic, out_col) +=
186 TFeat(interp_weights(j, k)) *
187 infeat(k, ic);
188 }
189 }
190 vec_valid_count = 0;
191 }
192 }
193
194 } // out_idx
195
196 if (out_importance) {
197 for (size_t out_idx = r.begin(); out_idx != r.end();
198 ++out_idx) {
199 const int out_col = out_idx - r.begin();
200 C.col(out_col) *= out_importance[out_idx];
201 }
202 }
203
204 Eigen::Matrix<TOut, Eigen::Dynamic, Eigen::Dynamic> A(
205 out_channels, spatial_filter_size * in_channels);
206
207 A = (C * B.transpose()).template cast<TOut>();
208
209 {
210 std::lock_guard<std::mutex> lock(filter_backprop_mutex);
211 int linear_i = 0;
212 for (int j = 0; j < spatial_filter_size * in_channels; ++j)
213 for (int i = 0; i < out_channels; ++i, ++linear_i) {
214 filter_backprop[linear_i] += A(i, j);
215 }
216 }
217 });
218}
219
302template <class TFeat, class TOut, class TReal, class TIndex>
303void CConvTransposeBackpropFilterCPU(TOut* filter_backprop,
304 const std::vector<int>& filter_dims,
305 size_t num_out,
306 const TReal* out_positions,
307 const TFeat* out_importance,
308 size_t num_inp,
309 const TReal* inp_positions,
310 const TFeat* inp_features,
311 const TFeat* inp_neighbors_importance_sum,
312 const int64_t* inp_neighbors_row_splits,
313 size_t neighbors_index_size,
314 const TIndex* neighbors_index,
315 const TFeat* neighbors_importance,
316 const int64_t* neighbors_row_splits,
317 const TReal* extents,
318 const TReal* offsets,
319 const TFeat* out_features_gradient,
320 InterpolationMode interpolation,
321 CoordinateMapping coordinate_mapping,
322 bool align_corners,
323 bool individual_extent,
324 bool isotropic_extent,
325 bool normalize) {
326#define FN_PARAMETERS \
327 filter_backprop, filter_dims, num_out, out_positions, out_importance, \
328 num_inp, inp_positions, inp_features, \
329 inp_neighbors_importance_sum, inp_neighbors_row_splits, \
330 neighbors_index_size, neighbors_index, neighbors_importance, \
331 neighbors_row_splits, extents, offsets, out_features_gradient
332
333#define CALL_TEMPLATE(INTERPOLATION, MAPPING, ALIGN_CORNERS, \
334 INDIVIDUAL_EXTENT, ISOTROPIC_EXTENT, NORMALIZE) \
335 if (INTERPOLATION == interpolation && MAPPING == coordinate_mapping && \
336 ALIGN_CORNERS == align_corners && \
337 INDIVIDUAL_EXTENT == individual_extent && \
338 ISOTROPIC_EXTENT == isotropic_extent && NORMALIZE == normalize) \
339 _CConvTransposeBackpropFilterCPU<TFeat, TOut, TReal, TIndex, \
340 INTERPOLATION, MAPPING, \
341 ALIGN_CORNERS, INDIVIDUAL_EXTENT, \
342 ISOTROPIC_EXTENT, NORMALIZE>( \
343 FN_PARAMETERS);
344
345#define CALL_TEMPLATE2(INTERPOLATION, MAPPING) \
346 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, true, true) \
347 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, true, false) \
348 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, false, true) \
349 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, true, false, false) \
350 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, true, true) \
351 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, true, false) \
352 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, false, true) \
353 CALL_TEMPLATE(INTERPOLATION, MAPPING, true, false, false, false) \
354 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, true, true) \
355 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, true, false) \
356 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, false, true) \
357 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, true, false, false) \
358 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, true, true) \
359 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, true, false) \
360 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, false, true) \
361 CALL_TEMPLATE(INTERPOLATION, MAPPING, false, false, false, false)
362
363#define CALL_TEMPLATE3(INTERPOLATION) \
364 CALL_TEMPLATE2(INTERPOLATION, CoordinateMapping::BALL_TO_CUBE_RADIAL) \
365 CALL_TEMPLATE2(INTERPOLATION, \
366 CoordinateMapping::BALL_TO_CUBE_VOLUME_PRESERVING) \
367 CALL_TEMPLATE2(INTERPOLATION, CoordinateMapping::IDENTITY)
368
369#define CALL_TEMPLATE4 \
370 CALL_TEMPLATE3(InterpolationMode::LINEAR) \
371 CALL_TEMPLATE3(InterpolationMode::LINEAR_BORDER) \
372 CALL_TEMPLATE3(InterpolationMode::NEAREST_NEIGHBOR)
373
375
376#undef CALL_TEMPLATE
377#undef CALL_TEMPLATE2
378#undef CALL_TEMPLATE3
379#undef CALL_TEMPLATE4
380
381#undef FN_PARAMETERS
382}
383
384} // namespace impl
385} // namespace ml
386} // namespace open3d
#define CALL_TEMPLATE4
#define VECSIZE
Eigen::Matrix3d B
Definition: PointCloudPlanarPatchDetection.cpp:506
void _CConvTransposeBackpropFilterCPU(TOut *filter_backprop, const std::vector< int > &filter_dims, size_t num_out, const TReal *out_positions, const TFeat *out_importance, size_t num_inp, const TReal *inp_positions, const TFeat *inp_features, const TFeat *inp_neighbors_importance_sum, const int64_t *inp_neighbors_row_splits, size_t neighbors_index_size, const TIndex *neighbors_index, const TFeat *neighbors_importance, const int64_t *neighbors_row_splits, const TReal *extents, const TReal *offsets, const TFeat *out_features_gradient)
Definition: ContinuousConvTransposeBackpropFilter.h:32
InterpolationMode
Definition: ContinuousConvTypes.h:18
void CConvTransposeBackpropFilterCPU(TOut *filter_backprop, const std::vector< int > &filter_dims, size_t num_out, const TReal *out_positions, const TFeat *out_importance, size_t num_inp, const TReal *inp_positions, const TFeat *inp_features, const TFeat *inp_neighbors_importance_sum, const int64_t *inp_neighbors_row_splits, size_t neighbors_index_size, const TIndex *neighbors_index, const TFeat *neighbors_importance, const int64_t *neighbors_row_splits, const TReal *extents, const TReal *offsets, const TFeat *out_features_gradient, InterpolationMode interpolation, CoordinateMapping coordinate_mapping, bool align_corners, bool individual_extent, bool isotropic_extent, bool normalize)
Definition: ContinuousConvTransposeBackpropFilter.h:303
CoordinateMapping
Definition: ContinuousConvTypes.h:26
Definition: PinholeCameraIntrinsic.cpp:16
Class for computing interpolation weights.
Definition: CoordinateTransformation.h:185