Use firebase push IDs for messages

client/src/js/actions/message.js

@@ -1,6 +1,8 @@
 import * as actions from '../actions';
 import { findBreakpoints, messageHeight, linkify, timestamp } from '../util';
 
+let nextID = 0;
+
 function initMessage(message, server, tab, state) {
   if (message.time) {
     message.time = timestamp(new Date(message.time * 1000));
@@ -8,6 +10,11 @@ function initMessage(message, server, tab, state) {
     message.time = timestamp();
   }
 
+  if (!message.id) {
+    message.id = nextID;
+    nextID++;
+  }
+
   if (tab.charAt(0) === '#') {
     message.channel = true;
   }

client/src/js/components/MessageBox.js

@@ -63,13 +63,14 @@ export default class MessageBox extends PureComponent {
     this.bottom = scrollTop + clientHeight >= scrollHeight;
   };
 
-  renderMessage = ({ index, style, key }) => {
+  renderMessage = ({ index, style }) => {
     const { messages, onNickClick } = this.props;
+    const message = messages.get(index);
 
     return (
       <Message
-        key={key}
+        key={message.id}
-        message={messages.get(index)}
+        message={message}
         style={style}
         onNickClick={onNickClick}
       />

server/irc_handler.go

@@ -4,6 +4,8 @@ import (
 	"log"
 	"strings"
 
+	"github.com/kjk/betterguid"
+
 	"github.com/khlieng/dispatch/irc"
 	"github.com/khlieng/dispatch/storage"
 )
@@ -122,6 +124,7 @@ func (i *ircHandler) mode(msg *irc.Message) {
 
 func (i *ircHandler) message(msg *irc.Message) {
 	message := Message{
+		ID:      betterguid.New(),
 		Server:  i.client.Host,
 		From:    msg.Nick,
 		Content: msg.LastParam(),
@@ -135,7 +138,8 @@ func (i *ircHandler) message(msg *irc.Message) {
 	}
 
 	if msg.Params[0] != "*" {
-		go i.session.user.LogMessage(i.client.Host, msg.Nick, msg.Params[0], msg.LastParam())
+		go i.session.user.LogMessage(message.ID,
+			i.client.Host, msg.Nick, msg.Params[0], msg.LastParam())
 	}
 }
 

server/irc_handler_test.go

@@ -130,12 +130,12 @@ func TestHandleIRCMessage(t *testing.T) {
 		Params:  []string{"#chan", "the message"},
 	})
 
-	checkResponse(t, "message", Message{
+	assert.Equal(t, "message", res.Type)
-		Server:  "host.com",
+	msg, ok := res.Data.(Message)
-		From:    "nick",
+	assert.True(t, ok)
-		To:      "#chan",
+	assert.Equal(t, "nick", msg.From)
-		Content: "the message",
+	assert.Equal(t, "#chan", msg.To)
-	}, res)
+	assert.Equal(t, "the message", msg.Content)
 
 	res = dispatchMessage(&irc.Message{
 		Command: irc.Privmsg,
@@ -143,11 +143,12 @@ func TestHandleIRCMessage(t *testing.T) {
 		Params:  []string{"nick", "the message"},
 	})
 
-	checkResponse(t, "pm", Message{
+	assert.Equal(t, "pm", res.Type)
-		Server:  "host.com",
+	msg, ok = res.Data.(Message)
-		From:    "someone",
+	assert.True(t, ok)
-		Content: "the message",
+	assert.Equal(t, "someone", msg.From)
-	}, res)
+	assert.Empty(t, msg.To)
+	assert.Equal(t, "the message", msg.Content)
 }
 
 func TestHandleIRCQuit(t *testing.T) {

server/json.go

@@ -61,6 +61,7 @@ type Quit struct {
 }
 
 type Message struct {
+	ID      string `json:"id"`
 	Server  string `json:"server"`
 	From    string `json:"from,omitempty"`
 	To      string `json:"to,omitempty"`

server/websocket_handler.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/gorilla/websocket"
+	"github.com/kjk/betterguid"
 	"github.com/spf13/viper"
 
 	"github.com/khlieng/dispatch/irc"
@@ -178,7 +179,9 @@ func (h *wsHandler) message(b []byte) {
 
 	if i, ok := h.session.getIRC(data.Server); ok {
 		i.Privmsg(data.To, data.Content)
-		go h.session.user.LogMessage(data.Server, i.GetNick(), data.To, data.Content)
+
+		go h.session.user.LogMessage(betterguid.New(),
+			data.Server, i.GetNick(), data.To, data.Content)
 	}
 }
 

storage/storage.schema

@@ -21,7 +21,7 @@ struct Channel {
 }
 
 struct Message {
-  ID      uint64
+  ID      string
   From    string
   Content string
   Time    int64

storage/storage.schema.gen.go

@@ -702,6 +702,21 @@ func (d *Channel) Unmarshal(buf []byte) (uint64, error) {
 
 func (d *Message) Size() (s uint64) {
 
+	{
+		l := uint64(len(d.ID))
+
+		{
+
+			t := l
+			for t >= 0x80 {
+				t >>= 7
+				s++
+			}
+			s++
+
+		}
+		s += l
+	}
 	{
 		l := uint64(len(d.From))
 
@@ -732,7 +747,7 @@ func (d *Message) Size() (s uint64) {
 		}
 		s += l
 	}
-	s += 16
+	s += 8
 	return
 }
 func (d *Message) Marshal(buf []byte) ([]byte, error) {
@@ -747,9 +762,23 @@ func (d *Message) Marshal(buf []byte) ([]byte, error) {
 	i := uint64(0)
 
 	{
+		l := uint64(len(d.ID))
 
-		*(*uint64)(unsafe.Pointer(&buf[0])) = d.ID
+		{
 
+			t := uint64(l)
+
+			for t >= 0x80 {
+				buf[i+0] = byte(t) | 0x80
+				t >>= 7
+				i++
+			}
+			buf[i+0] = byte(t)
+			i++
+
+		}
+		copy(buf[i+0:], d.ID)
+		i += l
 	}
 	{
 		l := uint64(len(d.From))
@@ -759,15 +788,15 @@ func (d *Message) Marshal(buf []byte) ([]byte, error) {
 			t := uint64(l)
 
 			for t >= 0x80 {
-				buf[i+8] = byte(t) | 0x80
+				buf[i+0] = byte(t) | 0x80
 				t >>= 7
 				i++
 			}
-			buf[i+8] = byte(t)
+			buf[i+0] = byte(t)
 			i++
 
 		}
-		copy(buf[i+8:], d.From)
+		copy(buf[i+0:], d.From)
 		i += l
 	}
 	{
@@ -778,43 +807,38 @@ func (d *Message) Marshal(buf []byte) ([]byte, error) {
 			t := uint64(l)
 
 			for t >= 0x80 {
-				buf[i+8] = byte(t) | 0x80
+				buf[i+0] = byte(t) | 0x80
 				t >>= 7
 				i++
 			}
-			buf[i+8] = byte(t)
+			buf[i+0] = byte(t)
 			i++
 
 		}
-		copy(buf[i+8:], d.Content)
+		copy(buf[i+0:], d.Content)
 		i += l
 	}
 	{
 
-		*(*int64)(unsafe.Pointer(&buf[i+8])) = d.Time
+		*(*int64)(unsafe.Pointer(&buf[i+0])) = d.Time
 
 	}
-	return buf[:i+16], nil
+	return buf[:i+8], nil
 }
 
 func (d *Message) Unmarshal(buf []byte) (uint64, error) {
 	i := uint64(0)
 
-	{
-
-		d.ID = *(*uint64)(unsafe.Pointer(&buf[i+0]))
-
-	}
 	{
 		l := uint64(0)
 
 		{
 
 			bs := uint8(7)
-			t := uint64(buf[i+8] & 0x7F)
+			t := uint64(buf[i+0] & 0x7F)
-			for buf[i+8]&0x80 == 0x80 {
+			for buf[i+0]&0x80 == 0x80 {
 				i++
-				t |= uint64(buf[i+8]&0x7F) << bs
+				t |= uint64(buf[i+0]&0x7F) << bs
 				bs += 7
 			}
 			i++
@@ -822,7 +846,7 @@ func (d *Message) Unmarshal(buf []byte) (uint64, error) {
 			l = t
 
 		}
-		d.From = string(buf[i+8 : i+8+l])
+		d.ID = string(buf[i+0 : i+0+l])
 		i += l
 	}
 	{
@@ -831,10 +855,10 @@ func (d *Message) Unmarshal(buf []byte) (uint64, error) {
 		{
 
 			bs := uint8(7)
-			t := uint64(buf[i+8] & 0x7F)
+			t := uint64(buf[i+0] & 0x7F)
-			for buf[i+8]&0x80 == 0x80 {
+			for buf[i+0]&0x80 == 0x80 {
 				i++
-				t |= uint64(buf[i+8]&0x7F) << bs
+				t |= uint64(buf[i+0]&0x7F) << bs
 				bs += 7
 			}
 			i++
@@ -842,13 +866,33 @@ func (d *Message) Unmarshal(buf []byte) (uint64, error) {
 			l = t
 
 		}
-		d.Content = string(buf[i+8 : i+8+l])
+		d.From = string(buf[i+0 : i+0+l])
+		i += l
+	}
+	{
+		l := uint64(0)
+
+		{
+
+			bs := uint8(7)
+			t := uint64(buf[i+0] & 0x7F)
+			for buf[i+0]&0x80 == 0x80 {
+				i++
+				t |= uint64(buf[i+0]&0x7F) << bs
+				bs += 7
+			}
+			i++
+
+			l = t
+
+		}
+		d.Content = string(buf[i+0 : i+0+l])
 		i += l
 	}
 	{
 
-		d.Time = *(*int64)(unsafe.Pointer(&buf[i+8]))
+		d.Time = *(*int64)(unsafe.Pointer(&buf[i+0]))
 
 	}
-	return i + 16, nil
+	return i + 8, nil
 }

storage/user_messages.go

@@ -2,8 +2,6 @@ package storage
 
 import (
 	"os"
-	"strconv"
-	"strings"
 	"time"
 
 	"github.com/blevesearch/bleve"
@@ -12,7 +10,7 @@ import (
 )
 
 type Message struct {
-	ID      uint64 `json:"id" bleve:"-"`
+	ID      string `json:"id" bleve:"-"`
 	Server  string `json:"-" bleve:"server"`
 	From    string `json:"from" bleve:"-"`
 	To      string `json:"-" bleve:"to"`
@@ -24,37 +22,35 @@ func (m Message) Type() string {
 	return "message"
 }
 
-func (u *User) LogMessage(server, from, to, content string) error {
+func (u *User) LogMessage(id, server, from, to, content string) error {
 	message := Message{
+		ID:      id,
 		Server:  server,
 		From:    from,
 		To:      to,
 		Content: content,
 		Time:    time.Now().Unix(),
 	}
-	bucketKey := server + ":" + to
 
 	err := u.messageLog.Batch(func(tx *bolt.Tx) error {
-		b, err := tx.Bucket(bucketMessages).CreateBucketIfNotExists([]byte(bucketKey))
+		b, err := tx.Bucket(bucketMessages).CreateBucketIfNotExists([]byte(server + ":" + to))
 		if err != nil {
 			return err
 		}
 
-		message.ID, _ = b.NextSequence()
-
 		data, err := message.Marshal(nil)
 		if err != nil {
 			return err
 		}
 
-		return b.Put(idToBytes(message.ID), data)
+		return b.Put([]byte(id), data)
 	})
 
 	if err != nil {
 		return err
 	}
 
-	return u.messageIndex.Index(bucketKey+":"+strconv.FormatUint(message.ID, 10), message)
+	return u.messageIndex.Index(id, message)
 }
 
 func (u *User) GetLastMessages(server, channel string, count int) ([]Message, error) {
@@ -85,7 +81,7 @@ func (u *User) GetLastMessages(server, channel string, count int) ([]Message, er
 	return nil, nil
 }
 
-func (u *User) GetMessages(server, channel string, count int, fromID uint64) ([]Message, error) {
+func (u *User) GetMessages(server, channel string, count int, fromID string) ([]Message, error) {
 	messages := make([]Message, count)
 
 	u.messageLog.View(func(tx *bolt.Tx) error {
@@ -95,7 +91,7 @@ func (u *User) GetMessages(server, channel string, count int, fromID uint64) ([]
 		}
 
 		c := b.Cursor()
-		c.Seek(idToBytes(fromID))
+		c.Seek([]byte(fromID))
 
 		for k, v := c.Prev(); count > 0 && k != nil; k, v = c.Prev() {
 			count--
@@ -134,15 +130,11 @@ func (u *User) SearchMessages(server, channel, q string) ([]Message, error) {
 
 	messages := []Message{}
 	u.messageLog.View(func(tx *bolt.Tx) error {
-		b := tx.Bucket(bucketMessages)
+		b := tx.Bucket(bucketMessages).Bucket([]byte(server + ":" + channel))
 
 		for _, hit := range searchResults.Hits {
-			idx := strings.LastIndex(hit.ID, ":")
-			bc := b.Bucket([]byte(hit.ID[:idx]))
-			id, _ := strconv.ParseUint(hit.ID[idx+1:], 10, 64)
-
 			message := Message{}
-			message.Unmarshal(bc.Get(idToBytes(id)))
+			message.Unmarshal(b.Get([]byte(hit.ID)))
 			messages = append(messages, message)
 		}
 

storage/user_test.go

@@ -6,6 +6,7 @@ import (
 	"strconv"
 	"testing"
 
+	"github.com/kjk/betterguid"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -87,7 +88,7 @@ func TestMessages(t *testing.T) {
 	user, err := NewUser()
 	assert.Nil(t, err)
 
-	messages, err := user.GetMessages("irc.freenode.net", "#go-nuts", 10, 6)
+	messages, err := user.GetMessages("irc.freenode.net", "#go-nuts", 10, "6")
 	assert.Nil(t, err)
 	assert.Len(t, messages, 0)
 
@@ -99,28 +100,31 @@ func TestMessages(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Len(t, messages, 0)
 
+	ids := []string{}
 	for i := 0; i < 5; i++ {
-		err = user.LogMessage("irc.freenode.net", "nick", "#go-nuts", "message"+strconv.Itoa(i))
+		id := betterguid.New()
+		ids = append(ids, id)
+		err = user.LogMessage(id, "irc.freenode.net", "nick", "#go-nuts", "message"+strconv.Itoa(i))
 		assert.Nil(t, err)
 	}
 
-	messages, err = user.GetMessages("irc.freenode.net", "#go-nuts", 10, 6)
+	messages, err = user.GetMessages("irc.freenode.net", "#go-nuts", 10, ids[4])
+	assert.Equal(t, "message0", messages[0].Content)
+	assert.Equal(t, "message3", messages[3].Content)
+	assert.Nil(t, err)
+	assert.Len(t, messages, 4)
+
+	messages, err = user.GetMessages("irc.freenode.net", "#go-nuts", 10, betterguid.New())
 	assert.Equal(t, "message0", messages[0].Content)
 	assert.Equal(t, "message4", messages[4].Content)
 	assert.Nil(t, err)
 	assert.Len(t, messages, 5)
 
-	messages, err = user.GetMessages("irc.freenode.net", "#go-nuts", 10, 100)
+	messages, err = user.GetMessages("irc.freenode.net", "#go-nuts", 10, ids[2])
 	assert.Equal(t, "message0", messages[0].Content)
-	assert.Equal(t, "message4", messages[4].Content)
+	assert.Equal(t, "message1", messages[1].Content)
 	assert.Nil(t, err)
-	assert.Len(t, messages, 5)
+	assert.Len(t, messages, 2)
-
-	messages, err = user.GetMessages("irc.freenode.net", "#go-nuts", 10, 4)
-	assert.Equal(t, "message0", messages[0].Content)
-	assert.Equal(t, "message2", messages[2].Content)
-	assert.Nil(t, err)
-	assert.Len(t, messages, 3)
 
 	messages, err = user.GetLastMessages("irc.freenode.net", "#go-nuts", 10)
 	assert.Equal(t, "message0", messages[0].Content)

vendor/github.com/kjk/betterguid/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Krzysztof Kowalczyk
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

vendor/github.com/kjk/betterguid/README.md

@@ -0,0 +1,26 @@
+This is Go package to generate guid (globally unique id) with good properties
+
+Usage:
+```go
+import "github.com/kjk/betterguid"
+
+id := betterguid.New()
+fmt.Printf("guid: '%s'\n", id)
+```
+
+Generated guids have good properties:
+* they're 20 character strings, safe for inclusion in urls (don't require escaping)
+* they're based on timestamp so that they sort **after** any existing ids
+* they contain 72-bits of random data after the timestamp so that IDs won't
+  collide with other clients' IDs
+* they sort **lexicographically** (so the timestamp is converted to characters
+  that will sort properly)
+* they're monotonically increasing.  Even if you generate more than one in the
+  same timestamp, thelatter ones will sort after the former ones.  We do this
+  by using the previous random bits but "incrementing" them by 1 (only in the
+  case of a timestamp collision).
+
+Read https://www.firebase.com/blog/2015-02-11-firebase-unique-identifiers.html
+for more info.
+
+Based on https://gist.github.com/mikelehen/3596a30bd69384624c11

vendor/github.com/kjk/betterguid/betterguid.go

@@ -0,0 +1,64 @@
+package betterguid
+
+import (
+	"math/rand"
+	"sync"
+	"time"
+)
+
+const (
+	// Modeled after base64 web-safe chars, but ordered by ASCII.
+	pushChars = "-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz"
+)
+
+var (
+	// Timestamp of last push, used to prevent local collisions if you push twice in one ms.
+	lastPushTimeMs int64
+	// We generate 72-bits of randomness which get turned into 12 characters and appended to the
+	// timestamp to prevent collisions with other clients.  We store the last characters we
+	// generated because in the event of a collision, we'll use those same characters except
+	// "incremented" by one.
+	lastRandChars [12]int
+	mu            sync.Mutex
+	rnd           *rand.Rand
+)
+
+func init() {
+	// have to seed to get randomness
+	rnd = rand.New(rand.NewSource(time.Now().UnixNano()))
+	for i := 0; i < 12; i++ {
+		lastRandChars[i] = rnd.Intn(64)
+	}
+}
+
+// New creates a new guid.
+func New() string {
+	var id [8 + 12]byte
+	mu.Lock()
+	timeMs := time.Now().UTC().UnixNano() / 1e6
+	if timeMs == lastPushTimeMs {
+		// increment lastRandChars
+		for i := 0; i < 12; i++ {
+			lastRandChars[i]++
+			if lastRandChars[i] < 64 {
+				break
+			}
+			// increment the next byte
+			lastRandChars[i] = 0
+		}
+	}
+	lastPushTimeMs = timeMs
+	// put random as the second part
+	for i := 0; i < 12; i++ {
+		id[19-i] = pushChars[lastRandChars[i]]
+	}
+	mu.Unlock()
+
+	// put current time at the beginning
+	for i := 7; i >= 0; i-- {
+		n := int(timeMs % 64)
+		id[i] = pushChars[n]
+		timeMs = timeMs / 64
+	}
+	return string(id[:])
+}

vendor/github.com/kjk/betterguid/doc.go

@@ -0,0 +1,22 @@
+/*
+
+Package betterguid generates 20-character guid (globally unique id) strings
+with good properties:
+
+They're 20 character strings, safe for inclusion in urls (don't require escaping)
+
+They're based on timestamp so that they sort **after** any existing ids
+
+They contain 72-bits of random data after the timestamp so that IDs won't collide with other clients' IDs
+
+They sort **lexicographically** (so the timestamp is converted to characters that will sort properly)
+
+They're monotonically increasing.  Even if you generate more than one in the same timestamp, thelatter ones will sort after the former ones.  We do this by using the previous random bits but "incrementing" them by 1 (only in the case of a timestamp collision).
+
+Read https://www.firebase.com/blog/2015-02-11-firebase-unique-identifiers.html
+for more info.
+
+Based on https://gist.github.com/mikelehen/3596a30bd69384624c11
+
+*/
+package betterguid

vendor/vendor.json

@@ -320,6 +320,12 @@
 			"revision": "06c7a16c845dc8e0bf575fafeeca0f5462f5eb4d",
 			"revisionTime": "2017-02-22T00:19:28Z"
 		},
+		{
+			"checksumSHA1": "kvX/+c8DWuAzUP2wdx1gB/MZbKY=",
+			"path": "github.com/kjk/betterguid",
+			"revision": "c70aca50d858c19f83b91cffcaeb50f0bc95488d",
+			"revisionTime": "2015-02-14T05:51:15Z"
+		},
 		{
 			"checksumSHA1": "UvboqgDDSAbGORdtr5tBpkwYR0A=",
 			"path": "github.com/magiconair/properties",