diff --git a/components/rgbd-sources/src/net.cpp b/components/rgbd-sources/src/net.cpp
index a1e84418c190ffc88e5ff66dced80fd24c2a146c..c9256adc6ff6afcdf4f0a292723631a35502b597 100644
--- a/components/rgbd-sources/src/net.cpp
+++ b/components/rgbd-sources/src/net.cpp
@@ -142,6 +142,8 @@ void NetSource::_recvChunk(int64_t frame, int chunk, bool delta, const vector<un
 		current_frame_ = frame;
 	} else if (frame < current_frame_) {
 		LOG(WARNING) << "Chunk dropped";
+		if (chunk == 0) N_--;
+		return;
 	}
 
 	// TODO:(Nick) Decode directly into double buffer if no scaling