imagepack: use emoji shortcode as fallback when importing packs

.gitignore

@@ -7,6 +7,7 @@
 
 *.json
 !pkg/connector/emojis/unicodemojipack.json
+!pkg/connector/emojis/shortcodes.json
 *.db*
 *.log
 *.bak

pkg/connector/emojis/emojis.go

@@ -20,9 +20,11 @@ import (
 	_ "embed"
 	"encoding/json"
 	"fmt"
+	"strings"
 	"sync"
 
 	"go.mau.fi/util/exstrings"
+	"go.mau.fi/util/variationselector"
 	"maunium.net/go/mautrix/bridgev2/networkid"
 	"maunium.net/go/mautrix/id"
 
@@ -36,6 +38,35 @@ var initOnce sync.Once
 var unicodemojiPack = map[string]int64{}
 var reverseUnicodemojiPack = map[int64]string{}
 
+//go:embed shortcodes.json
+//go:generate go run shortcode_gen.go
+var shortcodesJSON string
+var shortcodeMap = map[string]string{}
+var shortcodeInit = sync.OnceFunc(func() {
+	if err := json.Unmarshal(exstrings.UnsafeBytes(shortcodesJSON), &shortcodeMap); err != nil {
+		panic(fmt.Errorf("failed to unmarshal shortcodes: %w", err))
+	}
+})
+
+func GetShortcodes() map[string]string {
+	shortcodeInit()
+	return shortcodeMap
+}
+
+var skinToneRemover = strings.NewReplacer(
+	"\U0001F3FB", "",
+	"\U0001F3FC", "",
+	"\U0001F3FD", "",
+	"\U0001F3FE", "",
+	"\U0001F3FF", "",
+)
+
+func GetShortcode(emoji string) string {
+	sc := GetShortcodes()
+	emoji = skinToneRemover.Replace(emoji)
+	return sc[variationselector.Add(emoji)]
+}
+
 func doInit() {
 	if err := json.Unmarshal(exstrings.UnsafeBytes(unicodemojiPackJSON), &unicodemojiPack); err != nil {
 		panic(fmt.Errorf("Failed to unmarshal unicodemojipack: %w", err))

pkg/connector/emojis/shortcode_gen.go

@@ -0,0 +1,46 @@
+//go:build ignore
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+
+	"go.mau.fi/util/exerrors"
+	"go.mau.fi/util/variationselector"
+)
+
+type Emoji struct {
+	Unified   string `json:"unified"`
+	ShortName string `json:"short_name"`
+}
+
+func unifiedToUnicode(input string) string {
+	parts := strings.Split(input, "-")
+	output := make([]rune, len(parts))
+	for i, part := range parts {
+		output[i] = rune(exerrors.Must(strconv.ParseInt(part, 16, 32)))
+	}
+	return string(output)
+}
+
+func main() {
+	resp := exerrors.Must(http.Get("https://raw.githubusercontent.com/iamcal/emoji-data/master/emoji.json"))
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		panic(fmt.Errorf("unexpected status code %d", resp.StatusCode))
+	}
+	var emojis []*Emoji
+	exerrors.PanicIfNotNil(json.NewDecoder(resp.Body).Decode(&emojis))
+	output := make(map[string]string)
+	for _, emoji := range emojis {
+		output[variationselector.Add(unifiedToUnicode(emoji.Unified))] = emoji.ShortName
+	}
+	f := exerrors.Must(os.OpenFile("shortcodes.json", os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0644))
+	exerrors.PanicIfNotNil(json.NewEncoder(f).Encode(output))
+	_ = f.Close()
+}

pkg/connector/imagepack.go

@@ -43,6 +43,7 @@ import (
 	"maunium.net/go/mautrix/format"
 	"maunium.net/go/mautrix/id"
 
+	"go.mau.fi/mautrix-telegram/pkg/connector/emojis"
 	"go.mau.fi/mautrix-telegram/pkg/connector/media"
 	"go.mau.fi/mautrix-telegram/pkg/connector/store"
 	"go.mau.fi/mautrix-telegram/pkg/gotd/telegram/uploader"
@@ -496,14 +497,14 @@ func (tc *TelegramClient) fnDownloadEmojiPack(ce *commands.Event) {
 		},
 	}
 	keywords := make(map[int64][]string)
-	emojis := make(map[int64][]string)
+	emojiLists := make(map[int64][]string)
 	for _, kw := range set.Keywords {
 		keywords[kw.DocumentID] = kw.Keyword
 	}
 	for _, emojiPack := range set.Packs {
 		emoji := variationselector.Add(emojiPack.Emoticon)
 		for _, doc := range emojiPack.Documents {
-			emojis[doc] = append(emojis[doc], emoji)
+			emojiLists[doc] = append(emojiLists[doc], emoji)
 		}
 	}
 	evtID := ce.React("\u23f3\ufe0f")
@@ -520,7 +521,7 @@ func (tc *TelegramClient) fnDownloadEmojiPack(ce *commands.Event) {
 			return
 		}
 		kws := keywords[rawDoc.GetID()]
-		imageEmojis := emojis[rawDoc.GetID()]
+		imageEmojis := emojiLists[rawDoc.GetID()]
 		var key string
 		for _, kw := range kws {
 			_, alreadySet := pack.Images[kw]
@@ -530,12 +531,41 @@ func (tc *TelegramClient) fnDownloadEmojiPack(ce *commands.Event) {
 			key = kw
 			break
 		}
+		var firstShortcode string
+		if key == "" {
+			for _, emoji := range imageEmojis {
+				shortcode := emojis.GetShortcode(emoji)
+				if shortcode == "" {
+					continue
+				}
+				shortcode = fmt.Sprintf("%s_%s", set.Set.ShortName, shortcode)
+				if firstShortcode == "" {
+					firstShortcode = shortcode
+				}
+				_, alreadySet := pack.Images[shortcode]
+				if alreadySet {
+					continue
+				}
+				key = shortcode
+				break
+			}
+		}
+		if key == "" && firstShortcode != "" {
+			for i := 2; i < 10000; i++ {
+				kw := fmt.Sprintf("%s%d", firstShortcode, i)
+				_, alreadySet := pack.Images[kw]
+				if alreadySet {
+					continue
+				}
+				key = kw
+			}
+		}
 		if key == "" {
 			key = fmt.Sprintf("%s_img%d", set.Set.ShortName, i+1)
 		}
-		body := key
+		var emoji string
 		if len(imageEmojis) > 0 {
-			body = imageEmojis[0]
+			emoji = imageEmojis[0]
 		}
 		if !set.Set.Emojis {
 			// Stickers need extra info in each sticker so they can be accurately bridged back to Telegram
@@ -544,11 +574,12 @@ func (tc *TelegramClient) fnDownloadEmojiPack(ce *commands.Event) {
 				Network: StickerSourceID,
 				ID:      strconv.FormatInt(rawDoc.GetID(), 10),
 				PackURL: StickerPackURLPrefix + set.Set.ShortName,
+				Emoji:   emoji,
 			}
 		}
 		pack.Images[key] = &event.ImagePackImage{
 			URL:  mxc,
-			Body: body,
+			Body: cmp.Or(emoji, key),
 			Info: info,
 		}
 	}