Resolves #343 GUI and Frame Refactor

Major rewrite of the GUI into a module and view form, also using a new Feed class. The frame and frameset data structures have also been rewritten to support memory pooling and response frames.

components/structures/include/ftl/data/new_frame.hpp

 #ifndef _FTL_DATA_NEWFRAME_HPP_
 #define _FTL_DATA_NEWFRAME_HPP_
 
+// Remove pointless warning
+#ifdef _MSC_VER
+#pragma warning(disable : 4544)
+#endif
+
 #include <map>
 #include <unordered_set>
 #include <any>
 #include <optional>
 #include <list>
 #include <unordered_map>
+#include <functional>
 #include <ftl/codecs/channels.hpp>
+#include <ftl/codecs/packet.hpp>
 #include <ftl/data/channels.hpp>
 #include <ftl/exception.hpp>
 #include <ftl/handle.hpp>
+#include <ftl/data/messages.hpp>
 
 template<typename T> struct is_list : public std::false_type {};
 
@@ -23,16 +31,70 @@ namespace data {
 
 class Session;
 class Pool;
+class FrameSet;
+
+/**
+ * Unique identifier for a single frame. This is stored as two 16bit numbers
+ * packed into a 32bit int. Every frame has a FrameID, as does every frameset.
+ * FrameID + Timestamp together will be a unique object within the system since
+ * frames cannot be duplicated.
+ */
+struct FrameID {
+	uint32_t id;
+
+	/**
+	 * Frameset ID for this frame.
+	 */
+	inline unsigned int frameset() const { return id >> 8; }
+
+	/**
+	 * Frame index within the frameset. This will correspond to the vector
+	 * index in the frameset object.
+	 */
+	inline unsigned int source() const { return id & 0xff; }
+
+	/**
+	 * The packed int with both frameset ID and index.
+	 */
+	operator uint32_t() const { return id; }
 
+	/**
+	 * Create a frame ID using a frameset id and a source number.
+	 * @param fs Frameset id
+	 * @param s Source number inside frameset
+	 */
+	FrameID(unsigned int fs, unsigned int s) : id((fs << 8) + (s & 0xff) ) {}
+	FrameID() : id(0) {}
+};
+
+/**
+ * A frame object can be used in 3 different scenarios. A frame mode cannot be
+ * changed after construction and so each mode is constructed differently.
+ */
+enum FrameMode {
+	PRIMARY,		/// A normal frame generated by a builder
+	RESPONSE,		/// A frame that acts as a reply to a primary frame
+	STANDALONE		/// Not associated with a source or stream, used for storage
+};
+
+/**
+ * The life cycle of a frame goes through all of these frame status stages.
+ * From the `Pool` it is created. After the frame is populated with initial data
+ * it is `stored`. New data is inserted to the frame before being `flushed`.
+ * Finally, when the frame is destroyed the data is transfered to the `Pool`
+ * for memory reuse and the frame is `released`.
+ */
 enum FrameStatus {
-	CREATED,   // Initial state, before store
-	STORED,    // Changed to this after call to `store`
-	FLUSHED,   // Changed to this after call to `flush`
-	RELEASED   // Destroyed or moved
+	CREATED,   /// Initial state, before store
+	STORED,    /// Changed to this after call to `store`
+	FLUSHED,   /// Changed to this after call to `flush`
+	RELEASED   /// Destroyed or moved
 };
 
 /**
- * Helper class to enable aggregation of aggregate channels.
+ * Helper class to enable aggregation of aggregate channels. Assignment acts to
+ * append data to a list rather than replace that data. It allows all data
+ * changes to be recorded. Not thread-safe however.
  */
 template <typename T>
 struct Aggregator {
@@ -45,6 +107,16 @@ struct Aggregator {
 		return *this;
 	}
 
+	Aggregator &operator=(const typename T::value_type &v) {
+		list.push_back(v);
+		return *this;
+	}
+
+	Aggregator &operator=(typename T::value_type &&v) {
+		list.push_back(std::move(v));
+		return *this;
+	}
+
 	Aggregator &operator=(T &&l) {
 		if (aggregate) list.splice(list.end(), l, l.begin(), l.end());
 		else list = std::move(l);
@@ -55,24 +127,84 @@ struct Aggregator {
 	operator T() const { return list; }
 };
 
+/**
+ * A `Frame` is the primary unit of data within the system. A data source
+ * generates discrete blocks of data with a timestamp, these blocks are
+ * encapsulated in a frame that has any number of channels. A `Frame` must be
+ * constructed from a `Pool` object so that memory can be reused.
+ *
+ * It can be moved around but not copied since the quantity of data involved in
+ * a frame is huge.
+ *
+ * A frame goes through the following stages:
+ *   1) Creation from reused memory in `Pool`
+ *   2) Populate with incoming initial data/changes (from stream)
+ *   3) Store of changes to persistent memory
+ *   4) Create any new data such as new video frames
+ *   5) Flush the data to transmit or save, becomes readonly
+ *   6) Release memory to `Pool`
+ *
+ * A channel stores one particular element of data of a specified type. To write
+ * to a channel the `create` or `set` methods must be used, this will mark the
+ * channel as changed but can only occur before the frame is flushed and
+ * readonly. A `get` method allows const access to the data as long as the
+ * channel exists.
+ *
+ * On change events are triggered when `store` occurs, whereas on flush events
+ * occur after flush. Both of these may occur on destruction if the frame was
+ * not stored or flushed before destruction.
+ *
+ * Some channels may fail `hasChannel` but still be marked as `available`. This
+ * will be due to the data not being transmitted or encoded until requested.
+ *
+ * Each frame is also associated with a `Session` object which stores all
+ * persistent data. Persistent data can then be accessed via any `Frame` with
+ * the same ID since they share a `Session`.
+ *
+ * A `Frame` provides some basic methods, however, it can be cast to other
+ * frame types using the cast method which provides additional wrapper
+ * functions. An example is `ftl::rgbd::Frame`.
+ *
+ * @see https://gitlab.utu.fi/nicolas.pope/ftl/-/wikis/Design/Frames
+ */
 class Frame {
 	friend class Session;
 	friend class ftl::data::Pool;
-	friend class ftl::streams::Feed;
+	friend class ftl::data::FrameSet;
 
-	private:
+	protected:
 	// Only Feed class should construct
-	Frame(Pool *ppool, Session *parent, uint32_t pid, int64_t ts) : timestamp_(ts), id(pid), pool_(ppool), parent_(parent), status_(FrameStatus::CREATED) {};
+	Frame(Pool *ppool, Session *parent, FrameID pid, int64_t ts);
 	int64_t timestamp_=0;
+	FrameID id_;
 
 	public:
-	const uint32_t id=0;
 
+	/**
+	 * Millisecond timestamp when the frame was originally constructed and which
+	 * was the instant the data contents were captured.
+	 */
 	inline int64_t timestamp() const { return timestamp_; }
 
+	/**
+	 * Unique identification of data source. Combined with timestamp it will
+	 * become a unique item of data and a singleton in the system.
+	 */
+	inline FrameID id() const { return id_; }
+
+	/**
+	 * Access the frameset ID for this frame.
+	 */
+	inline unsigned int frameset() const { return id_.frameset(); }
+
+	/**
+	 * Access the index of the frame within the frameset.
+	 */
+	inline unsigned int source() const { return id_.source(); }
+
 	public:
 	Frame()=delete;
-	
+
 	~Frame();
 
 	Frame(Frame &&f) {
@@ -88,55 +220,197 @@ class Frame {
 	Frame(const Frame &)=delete;
 	Frame &operator=(const Frame &)=delete;
 
+	/**
+	 * Obtain the current life-cycle status of the frame. This determines what
+	 * operations are permitted and what the behviour of the frame is.
+	 */
 	inline FrameStatus status() const { return status_; }
 
+	/**
+	 * Number of data channels in the frame. Excluding previous persistent data.
+	 */
 	inline size_t size() const { return data_.size(); }
 
+	/**
+	 * Is there data in this frame or in the persistent store for the given
+	 * channel number?
+	 */
 	bool has(ftl::codecs::Channel c) const;
 
+	/**
+	 * Check that either this frame or the persistent store has all the
+	 * channels in the set.
+	 */
+	bool hasAll(const std::unordered_set<ftl::codecs::Channel> &cs);
+
+	/**
+	 * @see has(Channel)
+	 */
+	inline bool hasChannel(ftl::codecs::Channel c) const { return has(c); }
+
+	/**
+	 * Does this frame have data for a given channel. This excludes any data
+	 * that may be in the persistent store.
+	 */
 	inline bool hasOwn(ftl::codecs::Channel c) const;
 
+	/**
+	 * Is the channel potentially available if requested via a stream. Not all
+	 * channels are encoded and transmitted, but must be requested. This
+	 * indicates if such a request can be fullfilled.
+	 */
+	inline bool available(ftl::codecs::Channel c) const;
+
+	/**
+	 * A complete set of all channels that are potentially available but may
+	 * not currently have the data stored within this object. It means the
+	 * source of the frame can provide the data but has not be requested to
+	 * actually do so, or cannot due to resource constraints.
+	 */
+	std::unordered_set<ftl::codecs::Channel> available() const;
+
+	bool availableAll(const std::unordered_set<ftl::codecs::Channel> &cs) const;
+
+	/**
+	 * Used by a receiver to mark potential availability. Should not be used
+	 * elsewhere.
+	 */
+	inline void markAvailable(ftl::codecs::Channel c);
+
+	/**
+	 * Has a given channel been marked as changed?
+	 */
 	inline bool changed(ftl::codecs::Channel c) const;
 
+	/**
+	 * A channel is readonly if it has been flushed. An exception is thrown if
+	 * a write is attempted.
+	 */
 	inline bool readonly(ftl::codecs::Channel c) const;
 
+	/**
+	 * @see readonly(Channel)
+	 */
 	inline bool flushed(ftl::codecs::Channel c) const;
 
+	/**
+	 * Changes can occur from different sources for different reasons, this
+	 * obtains the cause of the change. For example, it can be a primary local
+	 * change or it can be received from a remote source. Change type does
+	 * influence behaviour during store and flush actions.
+	 */
 	inline ftl::data::ChangeType getChangeType(ftl::codecs::Channel c) const;
 
+	/**
+	 * Obtain the map of all changes.
+	 */
 	inline const std::unordered_map<ftl::codecs::Channel, ChangeType> &changed() const { return changed_; }
 
+	/**
+	 * Obtains a set of all available channels. This excludes channels in the
+	 * persistent store.
+	 */
+	std::unordered_set<ftl::codecs::Channel> channels() const;
+
+	/**
+	 * All channels including those in the persistent store.
+	 */
+	std::unordered_set<ftl::codecs::Channel> allChannels() const;
+
+	/**
+	 * Test if the type of the channel matches the template type. Other
+	 * functions throw exceptions if wrong type is used, but this will not. It
+	 * will also return false if the channel is missing.
+	 */
 	template <typename T>
 	bool isType(ftl::codecs::Channel c) const;
 
+	/**
+	 * Get a readonly const reference to the content of a channel. If the
+	 * channel does not exist or if the template type does not match the content
+	 * then it throws an exception. The data can either be within this frame,
+	 * or if not in the frame then it checks the persistent store.
+	 *
+	 * The data may internally still be encoded and will only be decoded on the
+	 * first call to `get`. It is therefore strongly advised that any initial
+	 * calls to `get` are not concurrent as it will not be thread-safe.
+	 */
 	template <typename T>
 	const T &get(ftl::codecs::Channel c) const;
 
+	/**
+	 * Should not be used directly, but allows direct access to the data for
+	 * a channel without any attempt to cast it to type. Throws exceptions if
+	 * the channel does not exist, but will also look in the persistent
+	 * store.
+	 */
 	const std::any &getAny(ftl::codecs::Channel c) const;
 
+	/**
+	 * Get the hash code from the C++ `type_info` structure that corresponds to
+	 * the current data contents.
+	 */
+	inline size_t type(ftl::codecs::Channel c) const { return getAny(c).type().hash_code(); }
+
+	/**
+	 * Should not be used. Allows modification without marking as changed.
+	 */
 	std::any &getAnyMutable(ftl::codecs::Channel c);
 
+	/**
+	 * Should not be used. Does not throw exceptions but can return a nullptr
+	 * instead if the channel does not exist or the type does not match.
+	 * Currently, this does not do lazy decode of data so may fail.
+	 */
 	template <typename T>
 	const T *getPtr(ftl::codecs::Channel c) const noexcept;
 
+	/**
+	 * Should not be used.
+	 */
 	template <typename T>
 	T *getMutable(ftl::codecs::Channel c);
 
+	/**
+	 * Mark a channel as changed even if there is no data. It can result in
+	 * `hasChannel` giving false but `changed` giving true. Intended to be used
+	 * internally.
+	 */
 	inline void touch(ftl::codecs::Channel c) {
-		changed_[c] = ChangeType::LOCAL;
+		markAvailable(c);
+		changed_[c] = (mode_ == FrameMode::PRIMARY) ? ChangeType::PRIMARY : ChangeType::RESPONSE;
 	}
 
+	/**
+	 * Should not be used.
+	 */
 	inline void touch(ftl::codecs::Channel c, ChangeType type) {
+		markAvailable(c);
 		changed_[c] = type;
 	}
 
+	/**
+	 * Mark the channel as unchanged. This will mean it will not be flushed,
+	 * transmitted or saved but will still return true with `hasChannel`.
+	 */
 	inline void untouch(ftl::codecs::Channel c) {
 		changed_.erase(c);
 	}
 
+	/**
+	 * Create a new channel with the given template type. It will mark the
+	 * channel as changed and return a mutable reference of the correct data
+	 * type. It is not possible to create a channel after it has been flushed,
+	 * this will throw an exception. The channel may have existing data from
+	 * the memory pool which can be overwritten, but this is not true for
+	 * every channel number (only video frames currently).
+	 */
 	template <typename T, std::enable_if_t<!is_list<T>::value,int> = 0>
 	T &create(ftl::codecs::Channel c);
 
+	/**
+	 * Create method used for aggregate channels. @see create.
+	 */
 	template <typename T, std::enable_if_t<is_list<T>::value,int> = 0>
 	ftl::data::Aggregator<T> create(ftl::codecs::Channel c);
 
@@ -145,8 +419,8 @@ class Frame {
 	 * changes the channel status to `DISPATCHED`. If the storage mode is
 	 * `persistent` this adds to session store instead of local frame store,
 	 * although the change status is added to the local frame.
-	 * 
-	 * To be used by receive, no one else.
+	 *
+	 * To be used by receiver, no one else. Currently unused.
 	 */
 	template <typename T, std::enable_if_t<!is_list<T>::value,int> = 0>
 	T &createChange(ftl::codecs::Channel c, ftl::data::ChangeType t);
@@ -154,14 +428,59 @@ class Frame {
 	template <typename T, std::enable_if_t<is_list<T>::value,int> = 0>
 	ftl::data::Aggregator<T> createChange(ftl::codecs::Channel c, ftl::data::ChangeType t);
 
+	/**
+	 * Create a change but with encoded data provided. This allows for both
+	 * lazy decode and for subsequent data forwarding without encoding.
+	 *
+	 * Currently unused.
+	 */
 	template <typename T>
-	T &createChange(ftl::codecs::Channel c, ftl::data::ChangeType t, std::vector<uint8_t> &data);
+	T &createChange(ftl::codecs::Channel c, ftl::data::ChangeType t, const ftl::codecs::Packet &data);
+
+	/**
+	 * Create a channel, mark with the given change type and provided encoded
+	 * data. Does not decode the data as it does not know the actually data
+	 * type of this channel at this time.
+	 *
+	 * To be used by `receiver`.
+	 * @see ftl::stream::Receiver
+	 */
+	inline void informChange(ftl::codecs::Channel c, ftl::data::ChangeType t, const ftl::codecs::Packet &data) {
+		createAnyChange(c, t, data);
+	}
+
+	/**
+	 * Create a channel, mark with a given change type but do not provide any
+	 * data or type information.
+	 */
+	inline void informChange(ftl::codecs::Channel c, ftl::data::ChangeType t) {
+		createAnyChange(c, t);
+	}
+
+	/**
+	 * Create a channel, mark with a given change type and provided unencoded
+	 * data. The data is moved into the channel. This is used by `Receiver` to
+	 * provide a loopback functionality.
+	 */
+	inline void informChange(ftl::codecs::Channel c, ftl::data::ChangeType t, std::any &data) {
+		createAnyChange(c, t) = std::move(data);
+	}
 
-	const std::vector<uint8_t> &getEncoded(ftl::codecs::Channel c) const;
+	/**
+	 * Retrieve all encoded data packets for a channel, if any. Note that
+	 * encoded data is removed if the channel is modified.
+	 */
+	const std::list<ftl::codecs::Packet> &getEncoded(ftl::codecs::Channel c) const;
 
+	/** Do not use. */
 	template <typename T, typename ...ARGS>
 	T &emplace(ftl::codecs::Channel, ARGS...);
 
+	/**
+	 * Can be used instead of `create` to modify channel contents. It has the
+	 * same rules as `create`, except that if the channel does not exist then
+	 * it will throw an exception instead of creating the channel.
+	 */
 	template <typename T, std::enable_if_t<!is_list<T>::value,int> = 0>
 	T &set(ftl::codecs::Channel c);
 
@@ -178,10 +497,29 @@ class Frame {
 	 */
 	void hardRemove(ftl::codecs::Channel);
 
+	/**
+	 * Add a callback to a channel to watch for change events. These are
+	 * triggered by the `store` operation. Note that `Receiver` will call
+	 * `store` on a frame before generating a frameset callback, therefore
+	 * these events always occur and complete before the frameset is generated.
+	 */
 	inline ftl::Handle onChange(ftl::codecs::Channel c, const std::function<bool(Frame&,ftl::codecs::Channel)> &cb);
 
+	/**
+	 * Add a callback to listen for any and all changes to the frame.
+	 * @see onChange(Channel, cb).
+	 */
 	inline ftl::Handle onChange(const std::function<bool(Frame&,ftl::codecs::Channel)> &cb);
 
+	/**
+	 * All changed channels generate a flush event when the frame is flushed
+	 * explicitly or on destruction. There is one exception, forwarded changes
+	 * do generate a change event but do no subsequently generate a flush event
+	 * as they are considered completed changes. This prevents loops whilst
+	 * ensuring everyone has a copy of the change.
+	 *
+	 * @see changeType
+	 */
 	inline ftl::Handle onFlush(const std::function<bool(Frame&,ftl::codecs::Channel)> &cb);
 
 	/**
@@ -196,6 +534,8 @@ class Frame {
 
 	void swapChannel(ftl::codecs::Channel, Frame &);
 
+	void swapChannels(ftl::codecs::Channel, ftl::codecs::Channel);
+
 	/**
 	 * Discard all change status without removing the data.
 	 */
@@ -218,35 +558,111 @@ class Frame {
 	 */
 	void release();
 
-	/** Send changes back through origin stream. */
+	/**
+	 * Send changes back through origin stream. Causes all channels to be
+	 * individually flushed, resulting in flush events and each channel being
+	 * readonly. Only changed channels are flushed. Note: A frame cannot be
+	 * flushed multiple times and the entire frame becomes readonly after this.
+	 */
 	bool flush();
 
+	/**
+	 * Force a flush of only a single channel, allowing the frame to continue
+	 * to be modified (except this channel). This will generate a single
+	 * flush event.
+	 */
 	bool flush(ftl::codecs::Channel c);
 
 	/** Copy persistent changes to session. To be called before dispatch. */
 	void store();
 
+	/**
+	 * Should only be used by Feed class. Ignores storage rules and saves
+	 * to session anyway. Unused.
+	 */
+	void forceStore();
+
+	/**
+	 * Iterator.
+	 */
 	inline auto begin() const { return data_.begin(); }
 	inline auto end() const { return data_.end(); }
 
-	// onBeginFlush
-	// onEndFlush
-	// onError
-
 	inline MUTEX &mutex();
 
+	/**
+	 * Generate a new frame to respond to this one. The destruction of this
+	 * new frame will flush the changes and results in those response changes
+	 * being transmitted back to the original source of the frame. The original
+	 * source will then see these changes in the next frame it attempt to
+	 * generate.
+	 */
+	Frame response() const;
+
+	/**
+	 * Convert this frame to another type. That type must not have any
+	 * additional member variables, only wrapper methods.
+	 */
+	template <typename T>
+	T &cast();
+
+	template <typename T>
+	const T &cast() const;
+
+	/**
+	 * Used to create isolated frame objects for buffer purposes. This is
+	 * deliberately separate from default constructor to force its explicit use.
+	 */
+	static Frame make_standalone();
+
+	/**
+	 * The memory pool associated with this frame. Note: the pool class also
+	 * provides `onFlush` events, allowing an event handler to respond to any
+	 * frame that is flushed.
+	 */
+	inline Pool *pool() const { return pool_; }
+
+	/**
+	 * The persistent data store for this frame. It is also a frame object and
+	 * can be used in the same manner.
+	 */
+	inline Session *parent() const { return parent_; }
+
+	inline FrameMode mode() const { return mode_; }
+
+	// ==== Wrapper functions ==================================================
+
+	void message(ftl::data::Message code, const std::string &msg);
+
+	void message(ftl::data::Message code, const ftl::Formatter &msg);
+
+	/** Note, throws exception if `Channel::Messages` is missing. */
+	const std::map<ftl::data::Message,std::string> &messages() const;
+
+	inline bool hasMessages() const { return hasChannel(ftl::codecs::Channel::Messages); }
+
+	/**
+	 * Get or generate a name for this frame.
+	 */
+	std::string name() const;
+
+	/** Can throw an exception if missing, use `hasChannel(Channel::MetaData)` first. */
+	const std::map<std::string,std::string> &metadata() const;
+
+	// =========================================================================
+
 	protected:
 	std::any &createAnyChange(ftl::codecs::Channel c, ftl::data::ChangeType t);
 
-	std::any &createAnyChange(ftl::codecs::Channel c, ftl::data::ChangeType t, std::vector<uint8_t> &data);
+	std::any &createAnyChange(ftl::codecs::Channel c, ftl::data::ChangeType t, const ftl::codecs::Packet &data);
 
 	std::any &createAny(ftl::codecs::Channel c);
 
 	private:
 	struct ChannelData {
-		ChannelStatus status=ChannelStatus::INVALID;
-		std::any data;
-		std::vector<uint8_t> encoded={};
+		mutable ChannelStatus status=ChannelStatus::INVALID;
+		mutable std::any data;
+		std::list<ftl::codecs::Packet> encoded={};
 	};
 
 	ChannelData &_getData(ftl::codecs::Channel);
@@ -257,18 +673,25 @@ class Frame {
 	Pool *pool_;
 	Session *parent_;
 	FrameStatus status_;
+	FrameMode mode_ = FrameMode::PRIMARY;
+	uint64_t available_ = 0;
 
 	inline void restart(int64_t ts) {
 		timestamp_ = ts;
 		status_ = FrameStatus::CREATED;
 	}
+
+	/**
+	 * Primary frames also store on flush.
+	 */
+	void _primaryStore();
 };
 
 class Session : public Frame {
 	friend class Frame;
 
 	public:
-	Session() : Frame(nullptr, nullptr,0,0) {
+	Session() : Frame(nullptr, nullptr,FrameID(0,0),0) {
 		status_ = FrameStatus::STORED;
 	}
 
@@ -290,7 +713,7 @@ class Session : public Frame {
 	void flush(Frame &f, ftl::codecs::Channel c);
 
 	inline MUTEX &mutex() { return mutex_; }
-	
+
 	private:
 	std::unordered_map<uint64_t, ftl::Handler<Frame&,ftl::codecs::Channel>> change_channel_;
 	ftl::Handler<Frame&,ftl::codecs::Channel> change_;
@@ -299,6 +722,25 @@ class Session : public Frame {
 	MUTEX mutex_;
 };
 
+template <typename T>
+bool make_type() {
+	setTypeEncoder(typeid(T).hash_code(), [](const ftl::data::Frame &f, ftl::codecs::Channel c, std::vector<uint8_t> &data) {
+		data.resize(0);
+		ftl::util::FTLVectorBuffer buf(data);
+		msgpack::pack(buf, f.get<T>(c));
+		return true;
+	});
+	return true;
+}
+
+template <typename T>
+bool decode_type(std::any &a, const std::vector<uint8_t> &data) {
+	auto unpacked = msgpack::unpack((const char*)data.data(), data.size());
+	T &t = a.emplace<T>();
+	unpacked.get().convert<T>(t);
+	return true;
+}
+
 }
 }
 
@@ -306,11 +748,34 @@ class Session : public Frame {
 
 MUTEX &ftl::data::Frame::mutex() { return parent_->mutex(); }
 
+template <typename T>
+T &ftl::data::Frame::cast() {
+	static_assert(std::is_base_of<Frame, T>::value, "Can only cast to type inheriting Frame");
+	static_assert(sizeof(T) == sizeof(Frame), "Casting type must not have additional data members");
+	return *reinterpret_cast<T*>(this);
+}
+
+template <typename T>
+const T &ftl::data::Frame::cast() const {
+	static_assert(std::is_base_of<Frame, T>::value, "Can only cast to type inheriting Frame");
+	static_assert(sizeof(T) == sizeof(Frame), "Casting type must not have additional data members");
+	return *reinterpret_cast<const T*>(this);
+}
+
 bool ftl::data::Frame::hasOwn(ftl::codecs::Channel c) const {
 	const auto &i = data_.find(c);
 	return (i != data_.end() && i->second.status != ftl::data::ChannelStatus::INVALID);
 }
 
+bool ftl::data::Frame::available(ftl::codecs::Channel c) const {
+	const int ic = static_cast<int>(c);
+	return (ic >= 64) ? has(c) : (0x1ull << ic) & available_;
+}
+
+void ftl::data::Frame::markAvailable(ftl::codecs::Channel c) {
+	if ((int)c < 64) available_ |= (0x1ull << (int)c);
+}
+
 bool ftl::data::Frame::changed(ftl::codecs::Channel c) const {
 	return changed_.find(c) != changed_.end();
 }
@@ -330,7 +795,7 @@ bool ftl::data::Frame::readonly(ftl::codecs::Channel c) const {
 }
 
 ftl::Handle ftl::data::Frame::onChange(ftl::codecs::Channel c, const std::function<bool(Frame&,ftl::codecs::Channel)> &cb) {
-	return parent_->onChange(id, c, cb);
+	return parent_->onChange(id(), c, cb);
 }
 
 ftl::Handle ftl::data::Frame::onChange(const std::function<bool(Frame&,ftl::codecs::Channel)> &cb) {
@@ -347,7 +812,7 @@ bool ftl::data::Frame::isType(ftl::codecs::Channel c) const {
 	if (i != data_.end() && i->second.status != ftl::data::ChannelStatus::INVALID) {
 		return typeid(T) == i->second.data.type();
 	} else {
-		return (parent_ && parent_->isType<T>(c)); 
+		return (parent_ && parent_->isType<T>(c));
 	}
 }
 
@@ -362,11 +827,27 @@ const T *ftl::data::Frame::getPtr(ftl::codecs::Channel c) const noexcept {
 template <typename T>
 const T &ftl::data::Frame::get(ftl::codecs::Channel c) const {
 	const auto &d = _getData(c);
+
+	if (d.status != ftl::data::ChannelStatus::INVALID && !d.data.has_value() && d.encoded.size() > 0) {
+		UNIQUE_LOCK(parent_->mutex(), lk);
+		if (!d.data.has_value()) {
+			// Do a decode now and change the status
+			d.status = ftl::data::ChannelStatus::DISPATCHED;
+
+			try {
+				decode_type<T>(d.data, d.encoded.front().data);
+			} catch (...) {
+				throw FTL_Error("Decode failure for channel " << int(c));
+			}
+		}
+	}
+
 	if (d.status != ftl::data::ChannelStatus::INVALID) {
+		if (!d.data.has_value()) throw FTL_Error("'get' does not have value (" << static_cast<unsigned int>(c) << ")");
 		auto *p = std::any_cast<T>(&d.data);
 		if (!p) throw FTL_Error("'get' wrong type for channel (" << static_cast<unsigned int>(c) << ")");
 		return *p;
-	} else throw FTL_Error("Missing channel (" << static_cast<unsigned int>(c) << ")");
+	} else throw FTL_Error("Missing channel (" << static_cast<unsigned int>(c) << ") for (" << frameset() << "," << source() << ")");
 }
 
 // Non-list version
@@ -375,6 +856,7 @@ T &ftl::data::Frame::create(ftl::codecs::Channel c) {
 	if (isAggregate(c)) throw FTL_Error("Aggregate channels must be of list type");
 
 	ftl::data::verifyChannelType<T>(c);
+	ftl::data::make_type<T>();
 
 	std::any &a = createAny(c);
 	if (!isType<T>(c)) return a.emplace<T>();
@@ -385,6 +867,7 @@ T &ftl::data::Frame::create(ftl::codecs::Channel c) {
 template <typename T, std::enable_if_t<is_list<T>::value,int> = 0>
 ftl::data::Aggregator<T> ftl::data::Frame::create(ftl::codecs::Channel c) {
 	ftl::data::verifyChannelType<T>(c);
+	ftl::data::make_type<T>();
 
 	std::any &a = createAny(c);
 	if (!isType<T>(c)) a.emplace<T>();
@@ -392,10 +875,11 @@ ftl::data::Aggregator<T> ftl::data::Frame::create(ftl::codecs::Channel c) {
 }
 
 template <typename T>
-T &ftl::data::Frame::createChange(ftl::codecs::Channel c, ftl::data::ChangeType type, std::vector<uint8_t> &data) {
+T &ftl::data::Frame::createChange(ftl::codecs::Channel c, ftl::data::ChangeType type, const ftl::codecs::Packet &data) {
 	if (!bool(is_list<T>{}) && isAggregate(c)) throw FTL_Error("Aggregate channels must be of list type");
 
 	ftl::data::verifyChannelType<T>(c);
+	//ftl::data::make_type<T>();
 
 	std::any &a = createAnyChange(c, type, data);
 	if (!isType<T>(c)) return a.emplace<T>();
@@ -408,6 +892,7 @@ T &ftl::data::Frame::createChange(ftl::codecs::Channel c, ftl::data::ChangeType
 	if (isAggregate(c)) throw FTL_Error("Aggregate channels must be of list type");
 
 	ftl::data::verifyChannelType<T>(c);
+	ftl::data::make_type<T>();
 
 	std::any &a = createAnyChange(c, type);
 	if (!isType<T>(c)) return a.emplace<T>();
@@ -418,6 +903,7 @@ T &ftl::data::Frame::createChange(ftl::codecs::Channel c, ftl::data::ChangeType
 template <typename T, std::enable_if_t<is_list<T>::value,int> = 0>
 ftl::data::Aggregator<T> ftl::data::Frame::createChange(ftl::codecs::Channel c, ftl::data::ChangeType type) {
 	ftl::data::verifyChannelType<T>(c);
+	ftl::data::make_type<T>();
 
 	std::any &a = createAnyChange(c, type);
 	if (!isType<T>(c)) a.emplace<T>();
@@ -474,4 +960,4 @@ ftl::data::Aggregator<T> ftl::data::Frame::set(ftl::codecs::Channel c) {
 	}
 }
 
-#endif
\ No newline at end of file
+#endif