Skip to content
Snippets Groups Projects
Commit fdb51c99 authored by Nicolas Pope's avatar Nicolas Pope
Browse files

WIP: Refactor of render code

parent 3316a577
No related branches found
No related tags found
1 merge request!123Implements #189 using density to estimate radius
Pipeline #15127 passed
......@@ -381,6 +381,7 @@ const GLTexture &ftl::gui::Camera::captureFrame() {
visualizeEnergy(depth_, tmp, 1.0);
texture_.update(tmp);
break;
case Channel::Density:
case Channel::Energy:
if (depth_.rows == 0) { break; }
visualizeEnergy(depth_, tmp, 10.0);
......
......@@ -191,6 +191,15 @@ MediaPanel::MediaPanel(ftl::gui::Screen *screen) : nanogui::Window(screen, ""),
}
});
button = new Button(popup, "Density");
button->setFlags(Button::RadioButton);
button->setCallback([this]() {
ftl::gui::Camera *cam = screen_->activeCamera();
if (cam) {
cam->setChannel(Channel::Density);
}
});
}
MediaPanel::~MediaPanel() {
......
......@@ -26,7 +26,7 @@ class Splatter : public ftl::render::Renderer {
//void setOutputDevice(int);
protected:
void renderChannel(ftl::render::SplatParams &params, ftl::rgbd::Frame &out, const ftl::rgbd::Channel &channel, cudaStream_t stream);
void _renderChannel(ftl::rgbd::Frame &out, const ftl::rgbd::Channel &channel, cudaStream_t stream);
private:
int device_;
......@@ -50,6 +50,12 @@ class Splatter : public ftl::render::Renderer {
float3 light_dir_;
uchar4 light_diffuse_;
uchar4 light_ambient_;
ftl::render::SplatParams params_;
template <typename T>
void __blendChannel(ftl::rgbd::Frame &, ftl::rgbd::Channel in, ftl::rgbd::Channel out, cudaStream_t);
void _blendChannel(ftl::rgbd::Frame &, ftl::rgbd::Channel in, ftl::rgbd::Channel out, cudaStream_t);
void _dibr(cudaStream_t);
};
}
......
......@@ -126,23 +126,73 @@ Splatter::~Splatter() {
}
void Splatter::renderChannel(
ftl::render::SplatParams &params, ftl::rgbd::Frame &out,
const Channel &channel, cudaStream_t stream)
{
if (channel == Channel::None) return;
template <typename T>
struct AccumSelector {
typedef float4 type;
static constexpr Channel channel = Channel::Colour;
//static constexpr cv::Scalar value = cv::Scalar(0.0f,0.0f,0.0f,0.0f);
};
template <>
struct AccumSelector<float> {
typedef float type;
static constexpr Channel channel = Channel::Colour2;
//static constexpr cv::Scalar value = cv::Scalar(0.0f);
};
template <typename T>
void Splatter::__blendChannel(ftl::rgbd::Frame &output, ftl::rgbd::Channel in, ftl::rgbd::Channel out, cudaStream_t stream) {
cv::cuda::Stream cvstream = cv::cuda::StreamAccessor::wrapStream(stream);
temp_.get<GpuMat>(Channel::Depth).setTo(cv::Scalar(0x7FFFFFFF), cvstream);
temp_.get<GpuMat>(Channel::Depth2).setTo(cv::Scalar(0x7FFFFFFF), cvstream);
temp_.get<GpuMat>(Channel::Colour).setTo(cv::Scalar(0.0f,0.0f,0.0f,0.0f), cvstream);
temp_.create<GpuMat>(
AccumSelector<T>::channel,
Format<typename AccumSelector<T>::type>(params_.camera.width, params_.camera.height)
).setTo(cv::Scalar(0.0f), cvstream);
temp_.get<GpuMat>(Channel::Contribution).setTo(cv::Scalar(0.0f), cvstream);
if (scene_->frames.size() < 1) return;
bool is_float = out.get<GpuMat>(channel).type() == CV_32F; //ftl::rgbd::isFloatChannel(channel);
bool is_4chan = out.get<GpuMat>(channel).type() == CV_32FC4;
temp_.createTexture<float>(Channel::Contribution);
for (auto &f : scene_->frames) {
if (f.get<GpuMat>(in).type() == CV_8UC3) {
// Convert to 4 channel colour
auto &col = f.get<GpuMat>(in);
GpuMat tmp(col.size(), CV_8UC4);
cv::cuda::swap(col, tmp);
cv::cuda::cvtColor(tmp,col, cv::COLOR_BGR2BGRA);
}
ftl::cuda::dibr_attribute(
f.createTexture<T>(in),
f.createTexture<float4>(Channel::Points),
temp_.getTexture<int>(Channel::Depth2),
temp_.createTexture<typename AccumSelector<T>::type>(AccumSelector<T>::channel),
temp_.getTexture<float>(Channel::Contribution),
params_, stream
);
}
ftl::cuda::dibr_normalise(
temp_.getTexture<typename AccumSelector<T>::type>(AccumSelector<T>::channel),
output.createTexture<T>(out),
temp_.getTexture<float>(Channel::Contribution),
stream
);
}
void Splatter::_blendChannel(ftl::rgbd::Frame &output, ftl::rgbd::Channel in, ftl::rgbd::Channel out, cudaStream_t stream) {
int type = output.get<GpuMat>(out).type(); // == CV_32F; //ftl::rgbd::isFloatChannel(channel);
switch (type) {
case CV_32F : __blendChannel<float>(output, in, out, stream); break;
case CV_32FC4 : __blendChannel<float4>(output, in, out, stream); break;
case CV_8UC4 : __blendChannel<uchar4>(output, in, out, stream); break;
default : LOG(ERROR) << "Invalid output channel format";
}
}
void Splatter::_dibr(cudaStream_t stream) {
cv::cuda::Stream cvstream = cv::cuda::StreamAccessor::wrapStream(stream);
temp_.get<GpuMat>(Channel::Depth2).setTo(cv::Scalar(0x7FFFFFFF), cvstream);
// Render each camera into virtual view
// TODO: Move out of renderChannel, this is a common step to all channels
for (size_t i=0; i < scene_->frames.size(); ++i) {
auto &f = scene_->frames[i];
auto *s = scene_->sources[i];
......@@ -156,129 +206,54 @@ void Splatter::renderChannel(
f.createTexture<float4>(Channel::Points),
f.createTexture<float4>(Channel::Normals),
temp_.createTexture<int>(Channel::Depth2),
params, backcull_, stream
params_, backcull_, stream
);
//LOG(INFO) << "DIBR DONE";
}
}
temp_.createTexture<float4>(Channel::Colour);
temp_.createTexture<float>(Channel::Contribution);
out.create<GpuMat>(Channel::Normals, Format<float4>(params.camera.width, params.camera.height));
out.get<GpuMat>(Channel::Normals).setTo(cv::Scalar(0.0f,0.0f,0.0f,0.0f), cvstream);
// Create normals first
for (auto &f : scene_->frames) {
void Splatter::_renderChannel(
ftl::rgbd::Frame &out,
const Channel &channel, cudaStream_t stream)
{
if (channel == Channel::None) return;
cv::cuda::Stream cvstream = cv::cuda::StreamAccessor::wrapStream(stream);
ftl::cuda::dibr_attribute(
f.createTexture<float4>(Channel::Normals),
f.createTexture<float4>(Channel::Points),
temp_.getTexture<int>(Channel::Depth2),
out.createTexture<float4>(Channel::Normals),
temp_.getTexture<float>(Channel::Contribution),
params, stream
);
}
if (scene_->frames.size() < 1) return;
bool is_float = out.get<GpuMat>(channel).type() == CV_32F; //ftl::rgbd::isFloatChannel(channel);
bool is_4chan = out.get<GpuMat>(channel).type() == CV_32FC4;
ftl::cuda::dibr_normalise(
out.getTexture<float4>(Channel::Normals),
out.getTexture<float4>(Channel::Normals),
temp_.getTexture<float>(Channel::Contribution),
stream
);
//auto &t = out.createTexture<float4>(Channel::Points, Format<float4>(params.camera.width, params.camera.height));
//ftl::cuda::point_cloud(t, temp_.getTexture<int>(Channel::Depth2), params.camera, params.m_viewMatrixInverse, 0, stream);
/*ftl::cuda::normals(out.createTexture<float4>(Channel::Normals),
temp_.getTexture<float4>(Channel::Normals),
temp_.getTexture<int>(Channel::Depth2),
9, 0.04f,
params.camera, params.m_viewMatrixInverse.getFloat3x3(), params.m_viewMatrix.getFloat3x3(), stream);*/
temp_.createTexture<float4>(Channel::Colour);
temp_.createTexture<float>(Channel::Contribution);
//if (norm_filter_ > -0.1f) {
// ftl::cuda::normal_filter(f.getTexture<float4>(Channel::Normals), f.getTexture<float4>(Channel::Points), s->parameters(), pose, norm_filter_, stream);
//}
// Generate initial normals for the splats
out.create<GpuMat>(Channel::Normals, Format<float4>(params_.camera.width, params_.camera.height));
_blendChannel(out, Channel::Normals, Channel::Normals, stream);
temp_.get<GpuMat>(Channel::Colour).setTo(cv::Scalar(0.0f,0.0f,0.0f,0.0f), cvstream);
temp_.get<GpuMat>(Channel::Contribution).setTo(cv::Scalar(0.0f), cvstream);
// Estimate point density
out.create<GpuMat>(Channel::Density, Format<float>(params_.camera.width, params_.camera.height));
_blendChannel(out, Channel::Depth, Channel::Density, stream);
// FIXME: Using colour 2 in this way seems broken since it is already used
if (is_4chan) {
temp_.create<GpuMat>(Channel::Colour2, Format<float4>(params.camera.width, params.camera.height));
temp_.create<GpuMat>(Channel::Colour2, Format<float4>(params_.camera.width, params_.camera.height));
temp_.get<GpuMat>(Channel::Colour2).setTo(cv::Scalar(0.0f,0.0f,0.0f,0.0f), cvstream);
} else if (is_float) {
temp_.create<GpuMat>(Channel::Colour2, Format<float>(params.camera.width, params.camera.height));
temp_.create<GpuMat>(Channel::Colour2, Format<float>(params_.camera.width, params_.camera.height));
temp_.get<GpuMat>(Channel::Colour2).setTo(cv::Scalar(0.0f), cvstream);
} else {
temp_.create<GpuMat>(Channel::Colour2, Format<uchar4>(params.camera.width, params.camera.height));
temp_.create<GpuMat>(Channel::Colour2, Format<uchar4>(params_.camera.width, params_.camera.height));
temp_.get<GpuMat>(Channel::Colour2).setTo(cv::Scalar(0,0,0,0), cvstream);
}
// Create attribute first
for (auto &f : scene_->frames) {
// Convert colour from BGR to BGRA if needed
if (f.get<GpuMat>(channel).type() == CV_8UC3) {
// Convert to 4 channel colour
auto &col = f.get<GpuMat>(Channel::Colour);
GpuMat tmp(col.size(), CV_8UC4);
cv::cuda::swap(col, tmp);
cv::cuda::cvtColor(tmp,col, cv::COLOR_BGR2BGRA);
}
if (is_4chan) {
ftl::cuda::dibr_attribute(
f.createTexture<float4>(channel),
f.createTexture<float4>(Channel::Points),
temp_.getTexture<int>(Channel::Depth2),
temp_.getTexture<float4>(Channel::Colour),
temp_.getTexture<float>(Channel::Contribution),
params, stream
);
} else if (is_float) {
ftl::cuda::dibr_attribute(
f.createTexture<float>(channel),
f.createTexture<float4>(Channel::Points),
temp_.getTexture<int>(Channel::Depth2),
temp_.createTexture<float>(Channel::Colour2),
temp_.getTexture<float>(Channel::Contribution),
params, stream
);
} else {
ftl::cuda::dibr_attribute(
f.createTexture<uchar4>(channel),
f.createTexture<float4>(Channel::Points),
temp_.getTexture<int>(Channel::Depth2),
temp_.createTexture<float4>(Channel::Colour),
temp_.getTexture<float>(Channel::Contribution),
params, stream
);
}
}
if (is_4chan) {
ftl::cuda::dibr_normalise(
temp_.getTexture<float4>(Channel::Colour),
(splat_) ? temp_.createTexture<float4>(Channel::Colour2) : out.createTexture<float4>(channel),
temp_.getTexture<float>(Channel::Contribution),
stream
);
} else if (is_float) {
ftl::cuda::dibr_normalise(
temp_.createTexture<float>(Channel::Colour2),
(splat_) ? temp_.createTexture<float>(Channel::Colour2) : out.createTexture<float>(channel),
temp_.getTexture<float>(Channel::Contribution),
stream
);
if (splat_) {
_blendChannel(temp_, channel, Channel::Colour2, stream);
} else {
ftl::cuda::dibr_normalise(
temp_.getTexture<float4>(Channel::Colour),
(splat_) ? temp_.createTexture<uchar4>(Channel::Colour2) : out.createTexture<uchar4>(channel),
temp_.getTexture<float>(Channel::Contribution),
stream
);
_blendChannel(out, channel, channel, stream);
}
//out.get<GpuMat>(Channel::Left).setTo(cv::Scalar(0,0,0,0), cvstream);
// Now splat the points
if (splat_) {
if (is_4chan) {
......@@ -288,7 +263,7 @@ void Splatter::renderChannel(
temp_.getTexture<int>(Channel::Depth2),
out.createTexture<float>(Channel::Depth),
out.createTexture<float4>(channel),
params, stream
params_, stream
);
} else if (is_float) {
ftl::cuda::splat(
......@@ -297,7 +272,7 @@ void Splatter::renderChannel(
temp_.getTexture<int>(Channel::Depth2),
out.createTexture<float>(Channel::Depth),
out.createTexture<float>(channel),
params, stream
params_, stream
);
} else {
ftl::cuda::splat(
......@@ -306,7 +281,7 @@ void Splatter::renderChannel(
temp_.getTexture<int>(Channel::Depth2),
out.createTexture<float>(Channel::Depth),
out.createTexture<uchar4>(channel),
params, stream
params_, stream
);
}
}
......@@ -334,7 +309,7 @@ bool Splatter::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out, cuda
cv::cuda::Stream cvstream = cv::cuda::StreamAccessor::wrapStream(stream);
// Parameters object to pass to CUDA describing the camera
SplatParams params;
SplatParams &params = params_;
params.m_flags = 0;
if (value("show_discontinuity_mask", false)) params.m_flags |= ftl::render::kShowDisconMask;
if (value("normal_weight_colours", true)) params.m_flags |= ftl::render::kNormalWeightColours;
......@@ -391,7 +366,8 @@ bool Splatter::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out, cuda
}
}
renderChannel(params, out, Channel::Colour, stream);
_dibr(stream);
_renderChannel(out, Channel::Colour, stream);
Channel chan = src->getChannel();
if (chan == Channel::Depth)
......@@ -401,7 +377,7 @@ bool Splatter::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out, cuda
out.create<GpuMat>(Channel::Normals, Format<float4>(camera.width, camera.height));
// Render normal attribute
renderChannel(params, out, Channel::Normals, stream);
_renderChannel(out, Channel::Normals, stream);
// Convert normal to single float value
temp_.create<GpuMat>(Channel::Colour, Format<uchar4>(camera.width, camera.height));
......@@ -427,7 +403,9 @@ bool Splatter::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out, cuda
out.create<GpuMat>(Channel::Right, Format<uchar4>(camera.width, camera.height));
out.get<GpuMat>(Channel::Right).setTo(background_, cvstream);
renderChannel(params, out, Channel::Right, stream);
_dibr(stream); // Need to re-dibr due to pose change
_renderChannel(out, Channel::Right, stream);
} else if (chan != Channel::None) {
if (ftl::rgbd::isFloatChannel(chan)) {
out.create<GpuMat>(chan, Format<float>(camera.width, camera.height));
......@@ -436,7 +414,7 @@ bool Splatter::render(ftl::rgbd::VirtualSource *src, ftl::rgbd::Frame &out, cuda
out.create<GpuMat>(chan, Format<uchar4>(camera.width, camera.height));
out.get<GpuMat>(chan).setTo(background_, cvstream);
}
renderChannel(params, out, chan, stream);
_renderChannel(out, chan, stream);
}
return true;
......
......@@ -25,6 +25,7 @@ enum struct Channel : int {
Flow, // 32F
Energy, // 32F
Mask, // 32U
Density, // 32F
LeftGray,
RightGray,
Overlay1
......@@ -106,6 +107,7 @@ inline bool isFloatChannel(ftl::rgbd::Channel chan) {
case Channel::Depth :
//case Channel::Normals :
case Channel::Confidence:
case Channel::Density:
case Channel::Energy : return true;
default : return false;
}
......
......@@ -187,6 +187,7 @@ ftl::cuda::TextureObject<T> &Frame::getTexture(ftl::rgbd::Channel c) {
auto &m = _get(c);
if (m.tex.cvType() != ftl::traits::OpenCVType<T>::value || m.tex.width() != m.gpu.cols || m.tex.height() != m.gpu.rows || m.gpu.type() != m.tex.cvType()) {
LOG(ERROR) << "Texture has not been created for channel = " << (int)c;
throw ftl::exception("Texture has not been created properly for this channel");
}
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment