From ae0088c94e14aca59befed78d6ba276304861a92 Mon Sep 17 00:00:00 2001
From: Nicolas Pope <nicolas.pope@utu.fi>
Date: Tue, 2 Jul 2019 10:04:03 +0300
Subject: [PATCH] Implements #125 second RGB channel
---
applications/gui/src/camera.cpp | 24 ++-
applications/gui/src/camera.hpp | 2 +-
applications/gui/src/media_panel.cpp | 28 ++-
applications/gui/src/media_panel.hpp | 4 +
applications/gui/src/screen.cpp | 8 +-
applications/gui/src/screen.hpp | 2 +
.../rgbd-sources/include/ftl/rgbd/source.hpp | 21 ++-
.../include/ftl/rgbd/streamer.hpp | 3 +
components/rgbd-sources/src/net.cpp | 43 ++++-
components/rgbd-sources/src/net.hpp | 2 +
components/rgbd-sources/src/source.cpp | 8 +-
components/rgbd-sources/src/stereovideo.cpp | 38 ++--
components/rgbd-sources/src/streamer.cpp | 177 +++++++++++-------
13 files changed, 261 insertions(+), 99 deletions(-)
diff --git a/applications/gui/src/camera.cpp b/applications/gui/src/camera.cpp
index b223e6f9c..5e1413ddb 100644
--- a/applications/gui/src/camera.cpp
+++ b/applications/gui/src/camera.cpp
@@ -230,6 +230,15 @@ void ftl::gui::Camera::showSettings() {
}
+void ftl::gui::Camera::setChannel(ftl::rgbd::channel_t c) {
+ channel_ = c;
+ switch (c) {
+ case ftl::rgbd::kChanRight:
+ case ftl::rgbd::kChanDepth: src_->setChannel(c); break;
+ default: src_->setChannel(ftl::rgbd::kChanNone);
+ }
+}
+
const GLTexture &ftl::gui::Camera::thumbnail() {
}
@@ -255,11 +264,13 @@ const GLTexture &ftl::gui::Camera::captureFrame() {
src_->grab();
src_->getFrames(rgb, depth);
- if (!stats_ && depth.rows > 0) {
- stats_ = new StatisticsImageNSamples(depth.size(), 25);
+ if (channel_ == ftl::rgbd::kChanDeviation) {
+ if (!stats_ && depth.rows > 0) {
+ stats_ = new StatisticsImageNSamples(depth.size(), 25);
+ }
+
+ if (stats_ && depth.rows > 0) { stats_->update(depth); }
}
-
- if (stats_ && depth.rows > 0) { stats_->update(depth); }
cv::Mat tmp;
@@ -282,6 +293,11 @@ const GLTexture &ftl::gui::Camera::captureFrame() {
texture_.update(tmp);
break;
+ case ftl::rgbd::kChanRight:
+ if (depth.rows == 0 || depth.type() != CV_8UC3) { break; }
+ texture_.update(depth);
+ break;
+
default:
if (rgb.rows == 0) { break; }
//imageSize = Vector2f(rgb.cols,rgb.rows);
diff --git a/applications/gui/src/camera.hpp b/applications/gui/src/camera.hpp
index 590bdbcee..5284b28eb 100644
--- a/applications/gui/src/camera.hpp
+++ b/applications/gui/src/camera.hpp
@@ -32,7 +32,7 @@ class Camera {
void showPoseWindow();
void showSettings();
- void setChannel(ftl::rgbd::channel_t c) { channel_ = c; };
+ void setChannel(ftl::rgbd::channel_t c);
void togglePause();
void isPaused();
diff --git a/applications/gui/src/media_panel.cpp b/applications/gui/src/media_panel.cpp
index 84b45e9fc..3d87de9e1 100644
--- a/applications/gui/src/media_panel.cpp
+++ b/applications/gui/src/media_panel.cpp
@@ -109,6 +109,7 @@ MediaPanel::MediaPanel(ftl::gui::Screen *screen) : nanogui::Window(screen, ""),
popbutton->setChevronIcon(ENTYPO_ICON_CHEVRON_SMALL_RIGHT);
Popup *popup = popbutton->popup();
popup->setLayout(new GroupLayout());
+ popup->setTheme(screen->toolbuttheme);
popup->setAnchorHeight(100);
button = new Button(popup, "Left");
@@ -121,9 +122,18 @@ MediaPanel::MediaPanel(ftl::gui::Screen *screen) : nanogui::Window(screen, ""),
}
});
- button = new Button(popup, "Depth");
- button->setFlags(Button::RadioButton);
- button->setCallback([this]() {
+ right_button_ = new Button(popup, "Right");
+ right_button_->setFlags(Button::RadioButton);
+ right_button_->setCallback([this]() {
+ ftl::gui::Camera *cam = screen_->activeCamera();
+ if (cam) {
+ cam->setChannel(ftl::rgbd::kChanRight);
+ }
+ });
+
+ depth_button_ = new Button(popup, "Depth");
+ depth_button_->setFlags(Button::RadioButton);
+ depth_button_->setCallback([this]() {
ftl::gui::Camera *cam = screen_->activeCamera();
if (cam) {
cam->setChannel(ftl::rgbd::kChanDepth);
@@ -143,3 +153,15 @@ MediaPanel::MediaPanel(ftl::gui::Screen *screen) : nanogui::Window(screen, ""),
MediaPanel::~MediaPanel() {
}
+
+// Update button enabled status
+void MediaPanel::cameraChanged() {
+ ftl::gui::Camera *cam = screen_->activeCamera();
+ if (cam) {
+ if (cam->source()->hasCapabilities(ftl::rgbd::kCapStereo)) {
+ right_button_->setEnabled(true);
+ } else {
+ right_button_->setEnabled(false);
+ }
+ }
+}
diff --git a/applications/gui/src/media_panel.hpp b/applications/gui/src/media_panel.hpp
index c65c48770..d7f9aa993 100644
--- a/applications/gui/src/media_panel.hpp
+++ b/applications/gui/src/media_panel.hpp
@@ -18,10 +18,14 @@ class MediaPanel : public nanogui::Window {
MediaPanel(ftl::gui::Screen *);
~MediaPanel();
+ void cameraChanged();
+
private:
ftl::gui::Screen *screen_;
bool paused_;
ftl::rgbd::SnapshotStreamWriter *writer_;
+ nanogui::Button *right_button_;
+ nanogui::Button *depth_button_;
};
}
diff --git a/applications/gui/src/screen.cpp b/applications/gui/src/screen.cpp
index 9f3d03bdf..e451e4819 100644
--- a/applications/gui/src/screen.cpp
+++ b/applications/gui/src/screen.cpp
@@ -62,7 +62,7 @@ ftl::gui::Screen::Screen(ftl::Configurable *proot, ftl::net::Universe *pnet, ftl
setSize(Vector2i(1280,720));
- Theme *toolbuttheme = new Theme(*theme());
+ toolbuttheme = new Theme(*theme());
toolbuttheme->mBorderDark = nanogui::Color(0,0);
toolbuttheme->mBorderLight = nanogui::Color(0,0);
toolbuttheme->mButtonGradientBotFocused = nanogui::Color(60,255);
@@ -71,8 +71,9 @@ ftl::gui::Screen::Screen(ftl::Configurable *proot, ftl::net::Universe *pnet, ftl
toolbuttheme->mButtonGradientTopUnfocused = nanogui::Color(0,0);
toolbuttheme->mButtonGradientTopPushed = nanogui::Color(60,180);
toolbuttheme->mButtonGradientBotPushed = nanogui::Color(60,180);
+ toolbuttheme->mTextColor = nanogui::Color(0.9f,0.9f,0.9f,0.9f);
- Theme *mediatheme = new Theme(*theme());
+ mediatheme = new Theme(*theme());
mediatheme->mIconScale = 1.2f;
mediatheme->mWindowDropShadowSize = 0;
mediatheme->mWindowFillFocused = nanogui::Color(45, 150);
@@ -255,11 +256,12 @@ void ftl::gui::Screen::setActiveCamera(ftl::gui::Camera *cam) {
if (cam) {
status_ = cam->source()->getURI();
mwindow_->setVisible(true);
+ mwindow_->cameraChanged();
swindow_->setVisible(false);
} else {
mwindow_->setVisible(false);
swindow_->setVisible(true);
- status_ = "No camera...";
+ status_ = "[No camera]";
}
}
diff --git a/applications/gui/src/screen.hpp b/applications/gui/src/screen.hpp
index 7746aa49e..8bfce830a 100644
--- a/applications/gui/src/screen.hpp
+++ b/applications/gui/src/screen.hpp
@@ -41,6 +41,8 @@ class Screen : public nanogui::Screen {
nanogui::Theme *windowtheme;
nanogui::Theme *specialtheme;
+ nanogui::Theme *mediatheme;
+ nanogui::Theme *toolbuttheme;
private:
ftl::gui::SourceWindow *swindow_;
diff --git a/components/rgbd-sources/include/ftl/rgbd/source.hpp b/components/rgbd-sources/include/ftl/rgbd/source.hpp
index 4fd875bc8..b084258bc 100644
--- a/components/rgbd-sources/include/ftl/rgbd/source.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/source.hpp
@@ -25,13 +25,14 @@ class SnapshotReader;
typedef unsigned int channel_t;
-static const channel_t kChanLeft = 1;
-static const channel_t kChanDepth = 2;
-static const channel_t kChanRight = 3;
-static const channel_t kChanDisparity = 4;
-static const channel_t kChanDeviation = 5;
+static const channel_t kChanNone = 0;
+static const channel_t kChanLeft = 0x0001;
+static const channel_t kChanDepth = 0x0002;
+static const channel_t kChanRight = 0x0004;
+static const channel_t kChanDisparity = 0x0008;
+static const channel_t kChanDeviation = 0x0010;
-static const channel_t kChanOverlay1 = 100;
+static const channel_t kChanOverlay1 = 0x1000;
/**
* RGBD Generic data source configurable entity. This class hides the
@@ -69,6 +70,13 @@ class Source : public ftl::Configurable {
*/
bool isReady() { return (impl_) ? impl_->isReady() : false; }
+ /**
+ * Change the second channel source.
+ */
+ bool setChannel(channel_t c);
+
+ channel_t getChannel() const { return channel_; }
+
/**
* Perform the hardware or virtual frame grab operation.
*/
@@ -180,6 +188,7 @@ class Source : public ftl::Configurable {
SHARED_MUTEX mutex_;
bool paused_;
bool bullet_;
+ channel_t channel_;
detail::Source *_createImplementation();
detail::Source *_createFileImpl(const ftl::URI &uri);
diff --git a/components/rgbd-sources/include/ftl/rgbd/streamer.hpp b/components/rgbd-sources/include/ftl/rgbd/streamer.hpp
index 2ab3bbda4..30631b7f9 100644
--- a/components/rgbd-sources/include/ftl/rgbd/streamer.hpp
+++ b/components/rgbd-sources/include/ftl/rgbd/streamer.hpp
@@ -118,6 +118,9 @@ class Streamer : public ftl::Configurable {
void _schedule();
void _swap(detail::StreamSource *);
void _addClient(const std::string &source, int N, int rate, const ftl::UUID &peer, const std::string &dest);
+ void _encodeAndTransmit(detail::StreamSource *src, int chunk);
+ void _encodeChannel1(const cv::Mat &in, std::vector<unsigned char> &out, unsigned int b);
+ bool _encodeChannel2(const cv::Mat &in, std::vector<unsigned char> &out, ftl::rgbd::channel_t c, unsigned int b);
};
}
diff --git a/components/rgbd-sources/src/net.cpp b/components/rgbd-sources/src/net.cpp
index bedac5b67..a5cc73e08 100644
--- a/components/rgbd-sources/src/net.cpp
+++ b/components/rgbd-sources/src/net.cpp
@@ -57,6 +57,7 @@ NetSource::NetSource(ftl::rgbd::Source *host)
gamma_ = host->value("gamma", 1.0f);
temperature_ = host->value("temperature", 6500);
+ default_quality_ = host->value("quality", 0);
host->on("gamma", [this,host](const ftl::config::Event&) {
gamma_ = host->value("gamma", 1.0f);
@@ -77,6 +78,10 @@ NetSource::NetSource(ftl::rgbd::Source *host)
host_->getNet()->send(peer_, "update_cfg", host_->getURI() + "/baseline", host_->getConfig()["baseline"].dump());
});
+ host->on("quality", [this,host](const ftl::config::Event&) {
+ default_quality_ = host->value("quality", 0);
+ });
+
_updateURI();
h_ = host_->getNet()->onConnect([this](ftl::net::Peer *p) {
@@ -101,7 +106,7 @@ void NetSource::_recvChunk(int frame, int chunk, bool delta, const vector<unsign
// Decode in temporary buffers to prevent long locks
cv::imdecode(jpg, cv::IMREAD_COLOR, &tmp_rgb);
- cv::imdecode(d, cv::IMREAD_UNCHANGED, &tmp_depth);
+ if (d.size() > 0) cv::imdecode(d, cv::IMREAD_UNCHANGED, &tmp_depth);
// Apply colour correction to chunk
ftl::rgbd::colourCorrection(tmp_rgb, gamma_, temperature_);
@@ -120,12 +125,25 @@ void NetSource::_recvChunk(int frame, int chunk, bool delta, const vector<unsign
// Original size so just copy
if (tmp_rgb.cols == chunkRGB.cols) {
tmp_rgb.copyTo(chunkRGB);
- tmp_depth.convertTo(chunkDepth, CV_32FC1, 1.0f/1000.0f); //(16.0f*10.0f));
+ if (!tmp_depth.empty() && tmp_depth.type() == CV_16U && chunkDepth.type() == CV_32F) {
+ tmp_depth.convertTo(chunkDepth, CV_32FC1, 1.0f/1000.0f); //(16.0f*10.0f));
+ } else if (!tmp_depth.empty() && tmp_depth.type() == CV_8UC3 && chunkDepth.type() == CV_8UC3) {
+ tmp_depth.copyTo(chunkDepth);
+ } else {
+ // Silent ignore?
+ }
// Downsized so needs a scale up
} else {
cv::resize(tmp_rgb, chunkRGB, chunkRGB.size());
tmp_depth.convertTo(tmp_depth, CV_32FC1, 1.0f/1000.0f);
- cv::resize(tmp_depth, chunkDepth, chunkDepth.size());
+ if (!tmp_depth.empty() && tmp_depth.type() == CV_16U && chunkDepth.type() == CV_32F) {
+ tmp_depth.convertTo(tmp_depth, CV_32FC1, 1.0f/1000.0f); //(16.0f*10.0f));
+ cv::resize(tmp_depth, chunkDepth, chunkDepth.size());
+ } else if (!tmp_depth.empty() && tmp_depth.type() == CV_8UC3 && chunkDepth.type() == CV_8UC3) {
+ cv::resize(tmp_depth, chunkDepth, chunkDepth.size());
+ } else {
+ // Silent ignore?
+ }
}
if (chunk == 0) {
@@ -150,6 +168,7 @@ void NetSource::setPose(const Eigen::Matrix4d &pose) {
void NetSource::_updateURI() {
UNIQUE_LOCK(mutex_,lk);
active_ = false;
+ prev_chan_ = ftl::rgbd::kChanNone;
auto uri = host_->get<string>("uri");
if (uri_.size() > 0) {
@@ -206,9 +225,25 @@ bool NetSource::grab(int n, int b) {
// Send k frames before end to prevent unwanted pause
// Unless only a single frame is requested
if ((N_ <= 2 && maxN_ > 1) || N_ == 0) {
+ const ftl::rgbd::channel_t chan = host_->getChannel();
+
N_ = maxN_;
- if (!host_->getNet()->send(peer_, "get_stream", *host_->get<string>("uri"), N_, minB_, host_->getNet()->id(), *host_->get<string>("uri"))) {
+ // Verify depth destination is of required type
+ if (chan == ftl::rgbd::kChanDepth && depth_.type() != CV_32F) {
+ depth_ = cv::Mat(cv::Size(params_.width, params_.height), CV_32FC1, 0.0f);
+ } else if (chan == ftl::rgbd::kChanRight && depth_.type() != CV_8UC3) {
+ depth_ = cv::Mat(cv::Size(params_.width, params_.height), CV_8UC3, cv::Scalar(0,0,0));
+ }
+
+ if (prev_chan_ != chan) {
+ host_->getNet()->send(peer_, "set_channel", *host_->get<string>("uri"), chan);
+ prev_chan_ = chan;
+ }
+
+ if (!host_->getNet()->send(peer_, "get_stream",
+ *host_->get<string>("uri"), N_, minB_,
+ host_->getNet()->id(), *host_->get<string>("uri"))) {
active_ = false;
}
diff --git a/components/rgbd-sources/src/net.hpp b/components/rgbd-sources/src/net.hpp
index a962d880a..3f8a86df3 100644
--- a/components/rgbd-sources/src/net.hpp
+++ b/components/rgbd-sources/src/net.hpp
@@ -45,6 +45,8 @@ class NetSource : public detail::Source {
int temperature_;
int minB_;
int maxN_;
+ int default_quality_;
+ ftl::rgbd::channel_t prev_chan_;
bool _getCalibration(ftl::net::Universe &net, const ftl::UUID &peer, const std::string &src, ftl::rgbd::Camera &p);
void _recv(const std::vector<unsigned char> &jpg, const std::vector<unsigned char> &d);
diff --git a/components/rgbd-sources/src/source.cpp b/components/rgbd-sources/src/source.cpp
index f810f38da..dbf23f203 100644
--- a/components/rgbd-sources/src/source.cpp
+++ b/components/rgbd-sources/src/source.cpp
@@ -132,7 +132,6 @@ ftl::rgbd::detail::Source *Source::_createFileImpl(const ftl::URI &uri) {
}
ftl::rgbd::detail::Source *Source::_createNetImpl(const ftl::URI &uri) {
- LOG(INFO) << "MAKE NET SOURCE";
return new NetSource(this);
}
@@ -203,6 +202,7 @@ capability_t Source::getCapabilities() const {
void Source::reset() {
UNIQUE_LOCK(mutex_,lk);
+ channel_ = kChanNone;
if (impl_) delete impl_;
impl_ = _createImplementation();
}
@@ -233,3 +233,9 @@ bool Source::thumbnail(cv::Mat &t) {
t = thumb_;
return !thumb_.empty();
}
+
+bool Source::setChannel(ftl::rgbd::channel_t c) {
+ channel_ = c;
+ // TODO(Nick) Verify channel is supported by this source...
+ return true;
+}
diff --git a/components/rgbd-sources/src/stereovideo.cpp b/components/rgbd-sources/src/stereovideo.cpp
index ad8fafa43..6e1c321a1 100644
--- a/components/rgbd-sources/src/stereovideo.cpp
+++ b/components/rgbd-sources/src/stereovideo.cpp
@@ -122,17 +122,33 @@ static void disparityToDepth(const cv::cuda::GpuMat &disparity, cv::cuda::GpuMat
}
bool StereoVideoSource::grab(int n, int b) {
- lsrc_->get(left_, right_, stream_);
- if (depth_tmp_.empty()) depth_tmp_ = cv::cuda::GpuMat(left_.size(), CV_32FC1);
- if (disp_tmp_.empty()) disp_tmp_ = cv::cuda::GpuMat(left_.size(), CV_32FC1);
- calib_->rectifyStereo(left_, right_, stream_);
- disp_->compute(left_, right_, disp_tmp_, stream_);
- disparityToDepth(disp_tmp_, depth_tmp_, calib_->getQ(), stream_);
- left_.download(rgb_, stream_);
- //rgb_ = lsrc_->cachedLeft();
- depth_tmp_.download(depth_, stream_);
-
- stream_.waitForCompletion();
+ const ftl::rgbd::channel_t chan = host_->getChannel();
+
+ if (chan == ftl::rgbd::kChanDepth) {
+ lsrc_->get(left_, right_, stream_);
+ if (depth_tmp_.empty()) depth_tmp_ = cv::cuda::GpuMat(left_.size(), CV_32FC1);
+ if (disp_tmp_.empty()) disp_tmp_ = cv::cuda::GpuMat(left_.size(), CV_32FC1);
+ calib_->rectifyStereo(left_, right_, stream_);
+ disp_->compute(left_, right_, disp_tmp_, stream_);
+ disparityToDepth(disp_tmp_, depth_tmp_, calib_->getQ(), stream_);
+ left_.download(rgb_, stream_);
+ //rgb_ = lsrc_->cachedLeft();
+ depth_tmp_.download(depth_, stream_);
+
+ stream_.waitForCompletion();
+ } else if (chan == ftl::rgbd::kChanRight) {
+ lsrc_->get(left_, right_, stream_);
+ calib_->rectifyStereo(left_, right_, stream_);
+ left_.download(rgb_, stream_);
+ right_.download(depth_, stream_);
+ stream_.waitForCompletion();
+ } else {
+ lsrc_->get(left_, right_, stream_);
+ calib_->rectifyStereo(left_, right_, stream_);
+ //rgb_ = lsrc_->cachedLeft();
+ left_.download(rgb_, stream_);
+ stream_.waitForCompletion();
+ }
return true;
}
diff --git a/components/rgbd-sources/src/streamer.cpp b/components/rgbd-sources/src/streamer.cpp
index 6c656ec79..815701cf4 100644
--- a/components/rgbd-sources/src/streamer.cpp
+++ b/components/rgbd-sources/src/streamer.cpp
@@ -91,6 +91,14 @@ Streamer::Streamer(nlohmann::json &config, Universe *net)
_addClient(source, N, rate, peer, dest);
});
+ net->bind("set_channel", [this](const string &uri, unsigned int chan) {
+ SHARED_LOCK(mutex_,slk);
+
+ if (sources_.find(uri) != sources_.end()) {
+ sources_[uri]->src->setChannel((ftl::rgbd::channel_t)chan);
+ }
+ });
+
net->bind("sync_streams", [this](unsigned long long time) {
// Calc timestamp delta
});
@@ -292,6 +300,7 @@ void Streamer::_schedule() {
// Grab job
ftl::pool.push([this,src](int id) {
//auto start = std::chrono::high_resolution_clock::now();
+
try {
src->src->grab();
} catch (std::exception &ex) {
@@ -319,72 +328,12 @@ void Streamer::_schedule() {
for (int i=0; i<kChunkCount; ++i) {
// Add chunk job to thread pool
ftl::pool.push([this,src](int id, int chunk) {
- if (!src->rgb.empty() && !src->depth.empty()) {
- bool delta = (chunk+src->frame) % 8 > 0; // Do XOR or not
- int chunk_width = src->rgb.cols / kChunkDim;
- int chunk_height = src->rgb.rows / kChunkDim;
-
- // Build chunk heads
- int cx = (chunk % kChunkDim) * chunk_width;
- int cy = (chunk / kChunkDim) * chunk_height;
- cv::Rect roi(cx,cy,chunk_width,chunk_height);
- vector<unsigned char> rgb_buf;
- cv::Mat chunkRGB = src->rgb(roi);
- cv::Mat chunkDepth = src->depth(roi);
- //cv::Mat chunkDepthPrev = src->prev_depth(roi);
-
- cv::Mat d2, d3;
- vector<unsigned char> d_buf;
- chunkDepth.convertTo(d2, CV_16UC1, 1000); // 16*10);
- //if (delta) d3 = (d2 * 2) - chunkDepthPrev;
- //else d3 = d2;
- //d2.copyTo(chunkDepthPrev);
-
- // For each allowed bitrate setting (0 = max quality)
- for (unsigned int b=0; b<10; ++b) {
- {
- //SHARED_LOCK(src->mutex,lk);
- if (src->clients[b].size() == 0) continue;
- }
-
- // Max bitrate means no changes
- if (b == 0) {
- cv::imencode(".jpg", chunkRGB, rgb_buf);
- vector<int> pngparams = {cv::IMWRITE_PNG_COMPRESSION, compress_level_}; // Default is 1 for fast, 9 = small but slow.
- cv::imencode(".png", d2, d_buf, pngparams);
-
- // Otherwise must downscale and change compression params
- // TODO(Nick) could reuse downscales
- } else {
- cv::Mat downrgb, downdepth;
- cv::resize(chunkRGB, downrgb, cv::Size(bitrate_settings[b].width / kChunkDim, bitrate_settings[b].height / kChunkDim));
- cv::resize(d2, downdepth, cv::Size(bitrate_settings[b].width / kChunkDim, bitrate_settings[b].height / kChunkDim));
- vector<int> jpgparams = {cv::IMWRITE_JPEG_QUALITY, bitrate_settings[b].jpg_quality};
- cv::imencode(".jpg", downrgb, rgb_buf, jpgparams);
- vector<int> pngparams = {cv::IMWRITE_PNG_COMPRESSION, bitrate_settings[b].png_compression}; // Default is 1 for fast, 9 = small but slow.
- cv::imencode(".png", downdepth, d_buf, pngparams);
- }
-
- //if (chunk == 0) LOG(INFO) << "Sending chunk " << chunk << " : size = " << (d_buf.size()+rgb_buf.size()) << "bytes";
-
- // Lock to prevent clients being added / removed
- SHARED_LOCK(src->mutex,lk);
- auto c = src->clients[b].begin();
- while (c != src->clients[b].end()) {
- try {
- // TODO(Nick) Send pose and timestamp
- if (!net_->send((*c).peerid, (*c).uri, 0, chunk, delta, rgb_buf, d_buf)) {
- // Send failed so mark as client stream completed
- (*c).txcount = (*c).txmax;
- } else {
- ++(*c).txcount;
- }
- } catch(...) {
- (*c).txcount = (*c).txmax;
- }
- ++c;
- }
- }
+ try {
+ if (!src->rgb.empty() && (src->src->getChannel() == ftl::rgbd::kChanNone || !src->depth.empty())) {
+ _encodeAndTransmit(src, chunk);
+ }
+ } catch(...) {
+ LOG(ERROR) << "Encode Exception: " << chunk;
}
src->jobs--;
@@ -397,6 +346,102 @@ void Streamer::_schedule() {
}
}
+void Streamer::_encodeAndTransmit(StreamSource *src, int chunk) {
+ bool hasChan2 = (!src->depth.empty() && src->src->getChannel() != ftl::rgbd::kChanNone);
+
+ bool delta = (chunk+src->frame) % 8 > 0; // Do XOR or not
+ int chunk_width = src->rgb.cols / kChunkDim;
+ int chunk_height = src->rgb.rows / kChunkDim;
+
+ // Build chunk heads
+ int cx = (chunk % kChunkDim) * chunk_width;
+ int cy = (chunk / kChunkDim) * chunk_height;
+ cv::Rect roi(cx,cy,chunk_width,chunk_height);
+ vector<unsigned char> rgb_buf;
+ cv::Mat chunkRGB = src->rgb(roi);
+ cv::Mat chunkDepth;
+ //cv::Mat chunkDepthPrev = src->prev_depth(roi);
+
+ cv::Mat d2, d3;
+ vector<unsigned char> d_buf;
+
+ if (hasChan2) {
+ chunkDepth = src->depth(roi);
+ if (chunkDepth.type() == CV_32F) chunkDepth.convertTo(d2, CV_16UC1, 1000); // 16*10);
+ else d2 = chunkDepth;
+ //if (delta) d3 = (d2 * 2) - chunkDepthPrev;
+ //else d3 = d2;
+ //d2.copyTo(chunkDepthPrev);
+ }
+
+ // For each allowed bitrate setting (0 = max quality)
+ for (unsigned int b=0; b<10; ++b) {
+ {
+ //SHARED_LOCK(src->mutex,lk);
+ if (src->clients[b].size() == 0) continue;
+ }
+
+ // Max bitrate means no changes
+ if (b == 0) {
+ _encodeChannel1(chunkRGB, rgb_buf, b);
+ if (hasChan2) _encodeChannel2(d2, d_buf, src->src->getChannel(), b);
+
+ // Otherwise must downscale and change compression params
+ // TODO:(Nick) could reuse downscales
+ } else {
+ cv::Mat downrgb, downdepth;
+ cv::resize(chunkRGB, downrgb, cv::Size(bitrate_settings[b].width / kChunkDim, bitrate_settings[b].height / kChunkDim));
+ if (hasChan2) cv::resize(d2, downdepth, cv::Size(bitrate_settings[b].width / kChunkDim, bitrate_settings[b].height / kChunkDim));
+
+ _encodeChannel1(downrgb, rgb_buf, b);
+ if (hasChan2) _encodeChannel2(downdepth, d_buf, src->src->getChannel(), b);
+ }
+
+ //if (chunk == 0) LOG(INFO) << "Sending chunk " << chunk << " : size = " << (d_buf.size()+rgb_buf.size()) << "bytes";
+
+ // Lock to prevent clients being added / removed
+ SHARED_LOCK(src->mutex,lk);
+ auto c = src->clients[b].begin();
+ while (c != src->clients[b].end()) {
+ try {
+ // TODO:(Nick) Send pose and timestamp
+ if (!net_->send((*c).peerid, (*c).uri, 0, chunk, delta, rgb_buf, d_buf)) {
+ // Send failed so mark as client stream completed
+ (*c).txcount = (*c).txmax;
+ } else {
+ ++(*c).txcount;
+ }
+ } catch(...) {
+ (*c).txcount = (*c).txmax;
+ }
+ ++c;
+ }
+ }
+}
+
+void Streamer::_encodeChannel1(const cv::Mat &in, vector<unsigned char> &out, unsigned int b) {
+ vector<int> jpgparams = {cv::IMWRITE_JPEG_QUALITY, bitrate_settings[b].jpg_quality};
+ cv::imencode(".jpg", in, out, jpgparams);
+}
+
+bool Streamer::_encodeChannel2(const cv::Mat &in, vector<unsigned char> &out, ftl::rgbd::channel_t c, unsigned int b) {
+ if (c == ftl::rgbd::kChanNone) return false; // NOTE: Should not happen
+
+ if (c == ftl::rgbd::kChanDepth && in.type() == CV_16U && in.channels() == 1) {
+ vector<int> params = {cv::IMWRITE_PNG_COMPRESSION, bitrate_settings[b].png_compression};
+ cv::imencode(".png", in, out, params);
+ return true;
+ } else if (c == ftl::rgbd::kChanRight && in.type() == CV_8UC3) {
+ vector<int> params = {cv::IMWRITE_JPEG_QUALITY, bitrate_settings[b].jpg_quality};
+ cv::imencode(".jpg", in, out, params);
+ return true;
+ } else {
+ LOG(ERROR) << "Bad channel configuration: channel=" << c << " imagetype=" << in.type();
+ }
+
+ return false;
+}
+
Source *Streamer::get(const std::string &uri) {
SHARED_LOCK(mutex_,slk);
if (sources_.find(uri) != sources_.end()) return sources_[uri]->src;
--
GitLab