Gateway working, beta

ws/handlers.go

@@ -2,7 +2,6 @@ package ws
 
 import (
 	"encoding/json"
-	"homestead/homestead_gateway/util/cache"
 	"net/http"
 	"time"
 
@@ -47,52 +46,53 @@ func (wsg *WebsocketGateway) handleSync(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	meta := cache.ConnectionMetaData{ConnectionType: handshake.Type}
-
 	switch handshake.Type {
 	case "mod":
 		var mhs ModHandshake
-
 		if err := json.Unmarshal(handshake.Data, &mhs); err != nil {
 			wsg.sendWebsocketError(conn, "Malformed mod handshake.", 400, true)
 			wsg.logger.Warn("Malformed mod handshake.", "remote", conn.RemoteAddr().String(), "err", err)
 			return
 		}
 
-		meta.ID = mhs.ServerID
-
-		if err = wsg.sendWebsocketResponse(conn, GatewayAck{Status: "connected", Type: "mod"}); err != nil {
+		if mhs.ServerID == "" || mhs.ChannelID == "" {
+			wsg.sendWebsocketError(conn, "Malformed mod handshake.", 400, true)
 			return
 		}
 
-		wsg.registerConn(conn, meta, "mod")
-		wsg.logger.Info("Mod connected via Websocket.", "remote", conn.RemoteAddr().String(), "server_id", mhs.ServerID)
+		if !wsg.registerConn(conn, "mod", mhs.ChannelID, mhs.ServerID) {
+			wsg.sendWebsocketError(conn, "Failed to register mod.", 500, true)
+			return
+		}
 
-		go wsg.read(conn, meta, "mod")
+		_ = wsg.sendWebsocketResponse(conn, GatewayAck{Status: "connected", Type: "mod"})
+		wsg.registry.FlushChannelWithSender(mhs.ChannelID, wsg.flush)
+
+		wsg.logger.Info("Mod connected via Websocket.", "remote", conn.RemoteAddr().String())
+		go wsg.read(conn, "mod", mhs.ChannelID)
 
 	case "bot":
 		var bhs BotHandshake
-
 		if err := json.Unmarshal(handshake.Data, &bhs); err != nil {
 			wsg.sendWebsocketError(conn, "Malformed bot handshake.", 400, true)
-			wsg.logger.Warn("Malformed bot handshake.", "remote", conn.RemoteAddr().String(), "err", err)
 			return
 		}
 
-		meta.ID = bhs.BotID
+		if bhs.ChannelId == "" {
+			wsg.sendWebsocketError(conn, "Malformed bot handshake.", 400, true)
+			return
+		}
 
-		if ok := wsg.registerConn(conn, meta, "bot"); !ok {
+		if !wsg.registerConn(conn, "bot", bhs.ChannelId, "") {
 			wsg.sendWebsocketError(conn, "Bot already connected.", 409, true)
 			return
 		}
 
-		if err = wsg.sendWebsocketResponse(conn, GatewayAck{Status: "connected", Type: "bot"}); err != nil {
-			return
-		}
+		_ = wsg.sendWebsocketResponse(conn, GatewayAck{Status: "connected", Type: "bot"})
+		wsg.registry.FlushAllToBotWithSender(wsg.flush)
 
-		wsg.logger.Info("Bot connected via Websocket.", "remote", conn.RemoteAddr().String(), "bot_id", bhs.BotID)
-
-		go wsg.read(conn, meta, "bot")
+		wsg.logger.Info("Bot connected via Websocket.", "remote", conn.RemoteAddr().String())
+		go wsg.read(conn, "bot", bhs.ChannelId)
 
 	default:
 		wsg.sendWebsocketError(conn, "Unknown handshake.", 400, true)

ws/registry.go

@@ -0,0 +1,219 @@
+package ws
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/gorilla/websocket"
+)
+
+func (q *BoundedQueue) Enqueue(m GatewayMessageOut) bool {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	if q.capacity == 0 {
+		return false
+	}
+	if q.length < q.capacity {
+		q.buf[(q.start+q.length)%q.capacity] = m
+		q.length++
+		return true
+	}
+	// overwrite oldest
+	q.buf[q.start] = m
+	q.start = (q.start + 1) % q.capacity
+	return true
+}
+
+func (q *BoundedQueue) PopAll() []GatewayMessageOut {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	if q.length == 0 {
+		return nil
+	}
+	out := make([]GatewayMessageOut, 0, q.length)
+	for i := 0; i < q.length; i++ {
+		out = append(out, q.buf[(q.start+i)%q.capacity])
+	}
+	q.start = 0
+	q.length = 0
+	return out
+}
+
+func (q *BoundedQueue) Len() int {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	return q.length
+}
+
+//
+
+func (r *Registry) getOrCreate(channel string) *ChannelEntry {
+	r.mu.RLock()
+	e := r.entries[channel]
+	r.mu.RUnlock()
+	if e != nil {
+		return e
+	}
+
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if e = r.entries[channel]; e == nil {
+		e = newChannelEntry(channel, r.queueCap)
+		r.entries[channel] = e
+	}
+	return e
+}
+
+//
+
+// RegisterMod : map channel_id -> mod conn (serverID)
+func (r *Registry) RegisterMod(channelID, serverID string, conn *websocket.Conn) {
+	e := r.getOrCreate(channelID)
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	if e.Mod != nil && e.Mod.Conn != nil {
+		_ = e.Mod.Conn.Close()
+	}
+	e.Mod = &ConnWrapper{Conn: conn, ServerID: serverID, LastSeen: time.Now()}
+	// flush queued bot->mod messages for this channel
+	// caller should use FlushChannelWithSender to perform actual sends
+}
+
+// UnregisterMod : remove mod for a channel
+func (r *Registry) UnregisterMod(channelID string) {
+	r.mu.RLock()
+	e := r.entries[channelID]
+	r.mu.RUnlock()
+	if e == nil {
+		return
+	}
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	if e.Mod != nil && e.Mod.Conn != nil {
+		_ = e.Mod.Conn.Close()
+	}
+	e.Mod = nil
+}
+
+// RegisterBot : single connection for bot. after registration call FlushAllToBotWithSender
+func (r *Registry) RegisterBot(conn *websocket.Conn) {
+	r.botMu.Lock()
+	if r.bot != nil && r.bot.Conn != nil {
+		_ = r.bot.Conn.Close()
+	}
+	r.bot = &ConnWrapper{Conn: conn, LastSeen: time.Now()}
+	r.botMu.Unlock()
+}
+
+func (r *Registry) UnregisterBot() {
+	r.botMu.Lock()
+	if r.bot != nil && r.bot.Conn != nil {
+		_ = r.bot.Conn.Close()
+	}
+	r.bot = nil
+	r.botMu.Unlock()
+}
+
+func (r *Registry) Send(channelID string, out GatewayMessageOut, sendOverConn func(*websocket.Conn, GatewayMessageOut) error) (delivered bool, queued bool, err error) {
+	if out.Type == "mod" {
+		r.botMu.Lock()
+		b := r.bot
+		r.botMu.Unlock()
+		if b != nil && b.Conn != nil {
+			if err := sendOverConn(b.Conn, out); err == nil {
+				return true, false, nil
+			}
+			_ = b.Conn.Close()
+			r.UnregisterBot()
+		}
+
+		e := r.getOrCreate(channelID)
+		e.mu.Lock()
+		enq := e.Queue.Enqueue(out)
+		e.mu.Unlock()
+		if !enq {
+			return false, false, fmt.Errorf("queue disabled")
+		}
+		return false, true, nil
+	}
+
+	e := r.getOrCreate(channelID)
+	e.mu.Lock()
+	mod := e.Mod
+	e.mu.Unlock()
+	if mod != nil && mod.Conn != nil {
+		if err := sendOverConn(mod.Conn, out); err == nil {
+			return true, false, nil
+		}
+		_ = mod.Conn.Close()
+		r.UnregisterMod(channelID)
+	}
+
+	e.mu.Lock()
+	enq := e.Queue.Enqueue(out)
+	e.mu.Unlock()
+	if !enq {
+		return false, false, fmt.Errorf("queue disabled")
+	}
+	return false, true, nil
+}
+
+//
+
+func (r *Registry) FlushChannelWithSender(channelID string, sendOverConn func(*websocket.Conn, GatewayMessageOut) error) {
+	r.mu.RLock()
+	e := r.entries[channelID]
+	r.mu.RUnlock()
+	if e == nil {
+		return
+	}
+	e.mu.Lock()
+	if e.Mod == nil || e.Mod.Conn == nil {
+		e.mu.Unlock()
+		return
+	}
+	msgs := e.Queue.PopAll()
+	modConn := e.Mod.Conn
+	e.mu.Unlock()
+
+	for _, m := range msgs {
+		if err := sendOverConn(modConn, m); err != nil {
+			// if send fails, re-enqueue (best-effort), drop-oldest logic applies
+			e.mu.Lock()
+			_ = e.Queue.Enqueue(m)
+			e.mu.Unlock()
+		}
+	}
+}
+
+func (r *Registry) FlushAllToBotWithSender(sendOverConn func(*websocket.Conn, GatewayMessageOut) error) {
+	r.botMu.Lock()
+	b := r.bot
+	r.botMu.Unlock()
+	if b == nil || b.Conn == nil {
+		return
+	}
+
+	r.mu.RLock()
+	entries := make([]*ChannelEntry, 0, len(r.entries))
+	for _, e := range r.entries {
+		entries = append(entries, e)
+	}
+	r.mu.RUnlock()
+
+	for _, e := range entries {
+		e.mu.Lock()
+		msgs := e.Queue.PopAll()
+		e.mu.Unlock()
+		if len(msgs) == 0 {
+			continue
+		}
+		for _, m := range msgs {
+			if err := sendOverConn(b.Conn, m); err != nil {
+				e.mu.Lock()
+				_ = e.Queue.Enqueue(m)
+				e.mu.Unlock()
+			}
+		}
+	}
+}

ws/structs.go

@@ -2,9 +2,8 @@ package ws
 
 import (
 	"encoding/json"
-	"homestead/homestead_gateway/util/cache"
-	"homestead/homestead_gateway/util/queue"
 	"log/slog"
+	"sync"
 	"time"
 
 	"github.com/gorilla/websocket"
@@ -17,16 +16,46 @@ type WebsocketGateway struct {
 
 	upgrader websocket.Upgrader
 
-	cache *cache.Cache
-	queue *queue.Queue[GatewayMessageOut]
-
-	bot   *websocket.Conn
-	conns *cache.ConnectionCache
+	registry *Registry
 
 	logger  *slog.Logger
 	closeFn func() error
 }
 
+//
+
+type Registry struct {
+	mu       sync.RWMutex
+	entries  map[string]*ChannelEntry
+	queueCap int
+
+	botMu sync.Mutex
+	bot   *ConnWrapper
+}
+
+type ConnWrapper struct {
+	Conn     *websocket.Conn
+	ServerID string // set for mods (the server_id)
+	LastSeen time.Time
+}
+
+type BoundedQueue struct {
+	mu       sync.Mutex
+	buf      []GatewayMessageOut
+	start    int
+	length   int
+	capacity int
+}
+
+type ChannelEntry struct {
+	mu      sync.Mutex
+	Channel string
+	Mod     *ConnWrapper
+	Queue   *BoundedQueue
+}
+
+//
+
 type User struct {
 	ID   string `json:"id"`
 	Name string `json:"name"`
@@ -64,15 +93,18 @@ type GatewayAck struct {
 	Type   string `json:"type"`
 }
 
+//
+
 type Handshake struct {
 	Type string          `json:"type"` // "mod" or "bot"
 	Data json.RawMessage `json:"data"`
 }
 
 type ModHandshake struct {
-	ServerID string `json:"server_id"`
+	ServerID  string `json:"server_id"`
+	ChannelID string `json:"channel_id"`
 }
 
 type BotHandshake struct {
-	BotID string `json:"bot_id"`
+	ChannelId string `json:"channel_id"`
 }

ws/temp.go

@@ -1,47 +0,0 @@
-package ws
-
-//type LoggingModHandler struct {
-//	logger *slog.Logger
-//}
-//
-//type LoggingBotHandler struct {
-//	logger *slog.Logger
-//}
-//
-//func NewLoggingModHandler(logger *slog.Logger) *LoggingModHandler {
-//	return &LoggingModHandler{logger: logger}
-//}
-//
-//func NewLoggingBotHandler(logger *slog.Logger) *LoggingBotHandler {
-//	return &LoggingBotHandler{logger: logger}
-//}
-//
-//func (h *LoggingModHandler) Handle(conn *websocket.Conn, msg GatewayMessageIn) error {
-//	// For now, just log and pretend it's being forwarded
-//	// TODO: Look up channel_id from database using server
-//	// TODO: Forward to bot connection(s)
-//
-//	fwd := GatewayMessageOut{
-//		Type:        "mod",
-//		ChannelID:   "TODO", // will come from database lookup
-//		Author:      msg.Author,
-//		Content:     msg.Content,
-//		Meta:        msg.Meta,
-//		Ts:          msg.Ts,
-//		ReceivedAt:  msg.ReceivedAt,
-//		ForwardedAt: time.Now().UTC(),
-//	}
-//
-//	_ = conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
-//
-//	if err := conn.WriteJSON(fwd); err != nil {
-//		_ = conn.Close()
-//		return err
-//	}
-//
-//	b, _ := json.Marshal(fwd)
-//	h.logger.Info("received mod message", "msg_id", msg.MsgID, "server", msg.Server, "Author", msg.Author.Name, "content", msg.Content)
-//	h.logger.Debug("forwarding mod message", "msg_id", msg.MsgID, "server", msg.Server, "payload", string(b))
-//
-//	return nil
-//}

ws/util.go

@@ -4,9 +4,9 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
-	"homestead/homestead_gateway/util/cache"
 	"log/slog"
 	"net/http"
+	"strings"
 	"time"
 
 	"github.com/gorilla/websocket"
@@ -36,6 +36,11 @@ func (wsg *WebsocketGateway) deafen(srv *http.Server) {
 
 // responses
 
+func (wsg *WebsocketGateway) flush(c *websocket.Conn, m GatewayMessageOut) error {
+	_ = c.SetWriteDeadline(time.Now().Add(5 * time.Second))
+	return c.WriteJSON(m)
+}
+
 func (wsg *WebsocketGateway) sendHttpError(w http.ResponseWriter, message string, code int) {
 	w.Header().Set("Content-Type", "application/json")
 	w.WriteHeader(code)
@@ -94,15 +99,6 @@ func (wsg *WebsocketGateway) validateApiKey(r *http.Request) bool {
 	return !(apiKey == "" || apiKey != wsg.apiKey)
 }
 
-func writeJSONSafe(c *websocket.Conn, v interface{}) error {
-	_ = c.SetWriteDeadline(time.Now().Add(5 * time.Second))
-	if err := c.WriteJSON(v); err != nil {
-		// caller handles logging
-		return err
-	}
-	return nil
-}
-
 func loggingMiddleware(logger *slog.Logger, next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		start := time.Now()
@@ -113,43 +109,112 @@ func loggingMiddleware(logger *slog.Logger, next http.Handler) http.Handler {
 
 // connections
 
-func (wsg *WebsocketGateway) registerConn(conn *websocket.Conn, meta cache.ConnectionMetaData, typ string) bool {
+func (wsg *WebsocketGateway) registerConn(conn *websocket.Conn, typ, channelId, serverId string) bool {
 	if typ == "bot" {
-		if wsg.bot != nil {
+		wsg.registry.botMu.Lock()
+		if wsg.registry.bot != nil && wsg.registry.bot.Conn != nil {
+			wsg.registry.botMu.Unlock()
 			return false
 		}
+		wsg.registry.botMu.Unlock()
 
-		wsg.bot = conn
+		wsg.registry.RegisterBot(conn)
 		return true
 	}
 
-	wsg.conns.Set(meta.ID, conn, &meta)
+	wsg.registry.RegisterMod(channelId, serverId, conn)
 	return true
 }
 
-func (wsg *WebsocketGateway) unregisterConn(conn *websocket.Conn, meta cache.ConnectionMetaData, typ string) {
+func (wsg *WebsocketGateway) unregisterConn(conn *websocket.Conn, typ, channelId string) {
 	if typ == "bot" {
-		_ = wsg.bot.Close()
-		wsg.bot = nil
+		wsg.registry.UnregisterBot()
+		if conn != nil {
+			_ = conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "Disconnecting."), time.Now().Add(time.Second))
+			_ = conn.Close()
+		}
 		return
 	}
 
-	wsg.conns.RemoveById(meta.ID)
-	_ = conn.Close()
+	wsg.registry.UnregisterMod(channelId)
+
+	if conn != nil {
+		_ = conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "Disconnecting."), time.Now().Add(time.Second))
+		_ = conn.Close()
+	}
 }
 
 func (wsg *WebsocketGateway) closeAll() {
 	wsg.logger.Info("Closing all websocket connections.")
 
-	if wsg.bot != nil {
-		_ = wsg.bot.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "Shutting down."), time.Now().Add(time.Second))
-		_ = wsg.bot.Close()
+	wsg.registry.UnregisterBot()
+
+	wsg.registry.mu.RLock()
+	entries := make([]*ChannelEntry, 0, len(wsg.registry.entries))
+	for _, e := range wsg.registry.entries {
+		entries = append(entries, e)
+	}
+	wsg.registry.mu.RUnlock()
+
+	for _, e := range entries {
+		e.mu.Lock()
+		modConn := e.Mod
+		if modConn != nil {
+			e.Mod = nil
+		}
+		e.mu.Unlock()
+
+		if modConn != nil && modConn.Conn != nil {
+			_ = modConn.Conn.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "Shutting down."), time.Now().Add(time.Second))
+			_ = modConn.Conn.Close()
+		}
+	}
+}
+
+//
+
+func NewRegistry(queueCap int) *Registry {
+	return &Registry{
+		entries:  make(map[string]*ChannelEntry),
+		queueCap: queueCap,
+	}
+}
+
+func NewBoundedQueue(cap int) *BoundedQueue {
+	if cap <= 0 {
+		cap = 128
+	}
+	return &BoundedQueue{buf: make([]GatewayMessageOut, cap), capacity: cap}
+}
+
+func newChannelEntry(channel string, cap int) *ChannelEntry {
+	return &ChannelEntry{
+		Channel: channel,
+		Queue:   NewBoundedQueue(cap),
+	}
+}
+
+//
+
+func (m *GatewayMessageIn) Validate() error {
+	if strings.TrimSpace(m.ID) == "" {
+		return errors.New("id missing")
+	}
+	if strings.TrimSpace(m.MsgID) == "" {
+		return errors.New("msg_id missing")
+	}
+	if strings.TrimSpace(m.Author.ID) == "" {
+		return errors.New("author.id missing")
+	}
+	if strings.TrimSpace(m.Content) == "" {
+		return errors.New("content missing")
 	}
 
-	wsg.conns.Range(func(id string, c *websocket.Conn, meta *cache.ConnectionMetaData) {
-		_ = c.WriteControl(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "Shutting down."), time.Now().Add(time.Second))
-		_ = c.Close()
-	})
+	if m.Type == "mod" && strings.TrimSpace(m.Destination.ID) == "" {
+		return errors.New("destination.channel_id missing")
+	}
 
-	wsg.conns.Clear()
+	return nil
 }
+
+func (c *ConnWrapper) Alive() bool { return c != nil && c.Conn != nil }

ws/validate.go

@@ -1,27 +0,0 @@
-package ws
-
-import (
-	"errors"
-	"strings"
-)
-
-func (m *GatewayMessageIn) Validate() error {
-	if strings.TrimSpace(m.ID) == "" {
-		return errors.New("id missing")
-	}
-	if strings.TrimSpace(m.MsgID) == "" {
-		return errors.New("msg_id missing")
-	}
-	if strings.TrimSpace(m.Author.ID) == "" {
-		return errors.New("author.id missing")
-	}
-	if strings.TrimSpace(m.Content) == "" {
-		return errors.New("content missing")
-	}
-
-	if m.Type == "mod" && strings.TrimSpace(m.Destination.ID) == "" {
-		return errors.New("destination.channel_id missing")
-	}
-
-	return nil
-}

ws/websocket.go

@@ -4,9 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"homestead/homestead_gateway/util/cache"
 	"homestead/homestead_gateway/util/config"
-	"homestead/homestead_gateway/util/queue"
 	"log/slog"
 	"net"
 	"net/http"
@@ -19,13 +17,11 @@ import (
 
 func NewWebsocketGateway(cfg config.GatewayConfig, logger *slog.Logger, closefn func() error) *WebsocketGateway {
 	return &WebsocketGateway{
-		logger:  logger,
-		closeFn: closefn,
-		port:    cfg.HttpPort,
-		apiKey:  cfg.Websocket,
-		cache:   cache.NewCache(),
-		queue:   queue.NewQueue[GatewayMessageOut](32),
-		conns:   cache.NewConnectionCache(),
+		logger:   logger,
+		closeFn:  closefn,
+		port:     cfg.HttpPort,
+		apiKey:   cfg.Websocket,
+		registry: NewRegistry(32),
 		upgrader: websocket.Upgrader{
 			ReadBufferSize:  1024,
 			WriteBufferSize: 1024,
@@ -184,9 +180,9 @@ func (wsg *WebsocketGateway) Serve(ctx context.Context, listenAddr string) error
 //	}
 //}
 
-func (wsg *WebsocketGateway) read(conn *websocket.Conn, meta cache.ConnectionMetaData, _type string) {
+func (wsg *WebsocketGateway) read(conn *websocket.Conn, _type, channelId string) {
 	defer func() {
-		wsg.unregisterConn(conn, meta, _type)
+		wsg.unregisterConn(conn, _type, channelId)
 		wsg.logger.Info("Client disconnected.", "remote", conn.RemoteAddr().String())
 	}()
 
@@ -235,70 +231,16 @@ func (wsg *WebsocketGateway) read(conn *websocket.Conn, meta cache.ConnectionMet
 			continue
 		}
 
-		var ok bool
-		var destConn *websocket.Conn
-
-		switch message.Type {
-		case "mod":
-			if wsg.bot != nil {
-				ok = true
-				destConn = wsg.bot
-			} else {
-				ok = false
-			}
-		case "bot":
-			var id string
-			id, ok = wsg.cache.GetByChannelId(message.ID)
-			if ok {
-				var dest *websocket.Conn
-				dest, _, ok = wsg.conns.GetById(id)
-				if ok {
-					destConn = dest
-				} else {
-					wsg.sendWebsocketError(conn, "Internal Server Error", 500, true)
-					wsg.logger.Error("Invalid cache structure.", "remote", conn.RemoteAddr().String(), "id", id)
-					return
-				}
-			}
-		default:
-			panic("invalid message type")
-		}
-
-		if ok {
-			if destConn == nil {
-				wsg.sendWebsocketError(conn, "Internal Server Error", 500, true)
-				wsg.logger.Error("Destination connection unavailable.", "remote", conn.RemoteAddr().String())
-				return
-			}
-
-			err = wsg.sendWebsocketResponse(destConn, GatewayMessageOut{
-				Type:        message.Type,
-				ID:          message.Destination.ID,
-				Author:      message.Author,
-				Content:     message.Content,
-				Meta:        message.Meta,
-				Ts:          message.Ts,
-				ReceivedAt:  message.ReceivedAt,
-				ForwardedAt: time.Now().UTC(),
-			})
-
-			if err != nil {
-				_ = wsg.sendWebsocketResponse(conn, GatewayAck{Status: "failed", Type: message.Type})
-				wsg.logger.Error("Failed to forward message.", "remote", conn.RemoteAddr().String(), "err", err)
-				continue
-			}
-
-			_ = wsg.sendWebsocketResponse(conn, GatewayAck{Status: "completed", Type: message.Type})
-			continue
-		}
-
-		if message.Type == "mod" {
-			wsg.cache.Set(message.ID, message.Destination.ID)
+		var outID string
+		if _type == "mod" {
+			outID = channelId
+		} else {
+			outID = message.ID
 		}
 
 		out := GatewayMessageOut{
 			Type:        message.Type,
-			ID:          message.Destination.ID,
+			ID:          outID,
 			Author:      message.Author,
 			Content:     message.Content,
 			Meta:        message.Meta,
@@ -307,13 +249,28 @@ func (wsg *WebsocketGateway) read(conn *websocket.Conn, meta cache.ConnectionMet
 			ForwardedAt: time.Now().UTC(),
 		}
 
-		queued := wsg.queue.Enqueue(out)
-		if !queued {
+		delivered, queued, err := wsg.registry.Send(out.ID, out, func(c *websocket.Conn, m GatewayMessageOut) error {
+			_ = c.SetWriteDeadline(time.Now().Add(5 * time.Second))
+			return c.WriteJSON(m)
+		})
+
+		if err != nil {
+			wsg.logger.Error("registry send error", "err", err)
 			_ = wsg.sendWebsocketResponse(conn, GatewayAck{Status: "failed", Type: message.Type})
-			wsg.logger.Warn("Failed to queue message.", "remote", conn.RemoteAddr().String())
 			continue
 		}
 
-		_ = wsg.sendWebsocketResponse(conn, GatewayAck{Status: "queued", Type: message.Type})
+		if delivered {
+			_ = wsg.sendWebsocketResponse(conn, GatewayAck{Status: "completed", Type: message.Type})
+			continue
+		}
+
+		if queued {
+			_ = wsg.sendWebsocketResponse(conn, GatewayAck{Status: "queued", Type: message.Type})
+			continue
+		}
+
+		_ = wsg.sendWebsocketResponse(conn, GatewayAck{Status: "failed", Type: message.Type})
+
 	}
 }