From 354cb2a6754b55fd3ad01388a4a17d3a76d4a09b Mon Sep 17 00:00:00 2001 From: dakkar Date: Sat, 9 Mar 2024 12:17:48 +0000 Subject: [PATCH] handle non-ASCII emoji names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * use the more inclusive regexp for validating emoji names * always normalize emoji names, aliases, categories the latter point is necessary to allow matching, for example, `รค` against `a`+combining diaeresis this will also need to bump the version of `sfm-js` once we merge https://activitypub.software/TransFem-org/sfm-js/-/merge_requests/2 --- .../ExportCustomEmojisProcessorService.ts | 2 +- .../ImportCustomEmojisProcessorService.ts | 13 +++++++------ .../endpoints/admin/emoji/add-aliases-bulk.ts | 2 +- .../src/server/api/endpoints/admin/emoji/add.ts | 11 ++++++----- .../server/api/endpoints/admin/emoji/copy.ts | 9 +++++---- .../api/endpoints/admin/emoji/list-remote.ts | 2 +- .../server/api/endpoints/admin/emoji/list.ts | 11 ++++++----- .../admin/emoji/remove-aliases-bulk.ts | 2 +- .../endpoints/admin/emoji/set-aliases-bulk.ts | 2 +- .../endpoints/admin/emoji/set-category-bulk.ts | 2 +- .../server/api/endpoints/admin/emoji/update.ts | 17 +++++++++-------- .../frontend/src/components/MkAutocomplete.vue | 2 +- .../frontend/src/components/MkEmojiPicker.vue | 2 +- .../frontend/src/pages/emoji-edit-dialog.vue | 2 +- packages/frontend/src/scripts/autocomplete.ts | 4 ++-- 15 files changed, 44 insertions(+), 39 deletions(-) diff --git a/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts b/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts index e4eb4791bd..45087927a5 100644 --- a/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportCustomEmojisProcessorService.ts @@ -85,7 +85,7 @@ export class ExportCustomEmojisProcessorService { }); for (const emoji of customEmojis) { - if (!/^[a-zA-Z0-9_]+$/.test(emoji.name)) { + if (!/^[\p{Letter}\p{Number}\p{Mark}_+-]+$/u.test(emoji.name)) { this.logger.error(`invalid emoji name: ${emoji.name}`); continue; } diff --git a/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts b/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts index 171809d25c..04ad74ee01 100644 --- a/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts +++ b/packages/backend/src/queue/processors/ImportCustomEmojisProcessorService.ts @@ -79,13 +79,14 @@ export class ImportCustomEmojisProcessorService { continue; } const emojiInfo = record.emoji; - if (!/^[a-zA-Z0-9_]+$/.test(emojiInfo.name)) { - this.logger.error(`invalid emojiname: ${emojiInfo.name}`); + const nameNfc = emojiInfo.name.normalize('NFC'); + if (!/^[\p{Letter}\p{Number}\p{Mark}_+-]+$/u.test(nameNfc)) { + this.logger.error(`invalid emojiname: ${nameNfc}`); continue; } const emojiPath = outputPath + '/' + record.fileName; await this.emojisRepository.delete({ - name: emojiInfo.name, + name: nameNfc, }); const driveFile = await this.driveService.addFile({ user: null, @@ -94,10 +95,10 @@ export class ImportCustomEmojisProcessorService { force: true, }); await this.customEmojiService.add({ - name: emojiInfo.name, - category: emojiInfo.category, + name: nameNfc, + category: emojiInfo.category?.normalize('NFC'), host: null, - aliases: emojiInfo.aliases, + aliases: emojiInfo.aliases?.map((a: string) => a.normalize('NFC')), driveFile, license: emojiInfo.license, isSensitive: emojiInfo.isSensitive, diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/add-aliases-bulk.ts b/packages/backend/src/server/api/endpoints/admin/emoji/add-aliases-bulk.ts index a30a080e59..f4fc79bdb3 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/add-aliases-bulk.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/add-aliases-bulk.ts @@ -34,7 +34,7 @@ export default class extends Endpoint { // eslint- private customEmojiService: CustomEmojiService, ) { super(meta, paramDef, async (ps, me) => { - await this.customEmojiService.addAliasesBulk(ps.ids, ps.aliases); + await this.customEmojiService.addAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC'))); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/add.ts b/packages/backend/src/server/api/endpoints/admin/emoji/add.ts index 767e517b80..b45a3c7156 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/add.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/add.ts @@ -40,7 +40,7 @@ export const meta = { export const paramDef = { type: 'object', properties: { - name: { type: 'string', pattern: '^[a-zA-Z0-9_]+$' }, + name: { type: 'string', pattern: '^[\\p{Letter}\\p{Number}\\p{Mark}_+-]+$' }, fileId: { type: 'string', format: 'misskey:id' }, category: { type: 'string', @@ -73,18 +73,19 @@ export default class extends Endpoint { // eslint- private emojiEntityService: EmojiEntityService, ) { super(meta, paramDef, async (ps, me) => { + const nameNfc = ps.name.normalize('NFC'); const driveFile = await this.driveFilesRepository.findOneBy({ id: ps.fileId }); if (driveFile == null) throw new ApiError(meta.errors.noSuchFile); - const isDuplicate = await this.customEmojiService.checkDuplicate(ps.name); + const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc); if (isDuplicate) throw new ApiError(meta.errors.duplicateName); if (driveFile.user !== null) await this.driveFilesRepository.update(driveFile.id, { user: null }); const emoji = await this.customEmojiService.add({ driveFile, - name: ps.name, - category: ps.category ?? null, - aliases: ps.aliases ?? [], + name: nameNfc, + category: ps.category?.normalize('NFC') ?? null, + aliases: ps.aliases?.map(a => a.normalize('NFC')) ?? [], host: null, license: ps.license ?? null, isSensitive: ps.isSensitive ?? false, diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts b/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts index 29af7598ed..f968813197 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/copy.ts @@ -82,15 +82,16 @@ export default class extends Endpoint { // eslint- throw new ApiError(); } + const nameNfc = emoji.name.normalize('NFC'); // Duplication Check - const isDuplicate = await this.customEmojiService.checkDuplicate(emoji.name); + const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc); if (isDuplicate) throw new ApiError(meta.errors.duplicateName); const addedEmoji = await this.customEmojiService.add({ driveFile, - name: emoji.name, - category: emoji.category, - aliases: emoji.aliases, + name: nameNfc, + category: emoji.category?.normalize('NFC'), + aliases: emoji.aliases?.map(a => a.normalize('NFC')), host: null, license: emoji.license, isSensitive: emoji.isSensitive, diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts b/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts index e423f440d0..1182918ea2 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts @@ -98,7 +98,7 @@ export default class extends Endpoint { // eslint- } if (ps.query) { - q.andWhere('emoji.name like :query', { query: '%' + sqlLikeEscape(ps.query) + '%' }) + q.andWhere('emoji.name like :query', { query: '%' + sqlLikeEscape(ps.query.normalize('NFC')) + '%' }) .orderBy('length(emoji.name)', 'ASC'); } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/list.ts b/packages/backend/src/server/api/endpoints/admin/emoji/list.ts index 53810d1d16..5e21111f9f 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/list.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/list.ts @@ -92,17 +92,18 @@ export default class extends Endpoint { // eslint- //const emojis = await q.limit(ps.limit).getMany(); emojis = await q.orderBy('length(emoji.name)', 'ASC').getMany(); - const queryarry = ps.query.match(/\:([a-z0-9_]*)\:/g); + const queryarry = ps.query.match(/:([\p{Letter}\p{Number}\p{Mark}_+-]*):/ug); if (queryarry) { emojis = emojis.filter(emoji => - queryarry.includes(`:${emoji.name}:`), + queryarry.includes(`:${emoji.name.normalize('NFC')}:`), ); } else { + const queryNfc = ps.query!.normalize('NFC'); emojis = emojis.filter(emoji => - emoji.name.includes(ps.query!) || - emoji.aliases.some(a => a.includes(ps.query!)) || - emoji.category?.includes(ps.query!)); + emoji.name.includes(queryNfc) || + emoji.aliases.some(a => a.includes(queryNfc)) || + emoji.category?.includes(queryNfc)); } emojis.splice(ps.limit + 1); } else { diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/remove-aliases-bulk.ts b/packages/backend/src/server/api/endpoints/admin/emoji/remove-aliases-bulk.ts index 0fa119eabe..e78620eac1 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/remove-aliases-bulk.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/remove-aliases-bulk.ts @@ -34,7 +34,7 @@ export default class extends Endpoint { // eslint- private customEmojiService: CustomEmojiService, ) { super(meta, paramDef, async (ps, me) => { - await this.customEmojiService.removeAliasesBulk(ps.ids, ps.aliases); + await this.customEmojiService.removeAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC'))); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/set-aliases-bulk.ts b/packages/backend/src/server/api/endpoints/admin/emoji/set-aliases-bulk.ts index d9ee18699c..69fc8e0cb5 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/set-aliases-bulk.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/set-aliases-bulk.ts @@ -34,7 +34,7 @@ export default class extends Endpoint { // eslint- private customEmojiService: CustomEmojiService, ) { super(meta, paramDef, async (ps, me) => { - await this.customEmojiService.setAliasesBulk(ps.ids, ps.aliases); + await this.customEmojiService.setAliasesBulk(ps.ids, ps.aliases.map(a => a.normalize('NFC'))); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/set-category-bulk.ts b/packages/backend/src/server/api/endpoints/admin/emoji/set-category-bulk.ts index dc25df2767..679a9f9c42 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/set-category-bulk.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/set-category-bulk.ts @@ -36,7 +36,7 @@ export default class extends Endpoint { // eslint- private customEmojiService: CustomEmojiService, ) { super(meta, paramDef, async (ps, me) => { - await this.customEmojiService.setCategoryBulk(ps.ids, ps.category ?? null); + await this.customEmojiService.setCategoryBulk(ps.ids, ps.category?.normalize('NFC') ?? null); }); } } diff --git a/packages/backend/src/server/api/endpoints/admin/emoji/update.ts b/packages/backend/src/server/api/endpoints/admin/emoji/update.ts index 22609a16a3..3caa0f84a3 100644 --- a/packages/backend/src/server/api/endpoints/admin/emoji/update.ts +++ b/packages/backend/src/server/api/endpoints/admin/emoji/update.ts @@ -40,7 +40,7 @@ export const paramDef = { type: 'object', properties: { id: { type: 'string', format: 'misskey:id' }, - name: { type: 'string', pattern: '^[a-zA-Z0-9_]+$' }, + name: { type: 'string', pattern: '^[\\p{Letter}\\p{Number}\\p{Mark}_+-]+$' }, fileId: { type: 'string', format: 'misskey:id' }, category: { type: 'string', @@ -72,6 +72,7 @@ export default class extends Endpoint { // eslint- private customEmojiService: CustomEmojiService, ) { super(meta, paramDef, async (ps, me) => { + const nameNfc = ps.name?.normalize('NFC'); let driveFile; if (ps.fileId) { driveFile = await this.driveFilesRepository.findOneBy({ id: ps.fileId }); @@ -83,22 +84,22 @@ export default class extends Endpoint { // eslint- emojiId = ps.id; const emoji = await this.customEmojiService.getEmojiById(ps.id); if (!emoji) throw new ApiError(meta.errors.noSuchEmoji); - if (ps.name && (ps.name !== emoji.name)) { - const isDuplicate = await this.customEmojiService.checkDuplicate(ps.name); + if (nameNfc && (nameNfc !== emoji.name)) { + const isDuplicate = await this.customEmojiService.checkDuplicate(nameNfc); if (isDuplicate) throw new ApiError(meta.errors.sameNameEmojiExists); } } else { - if (!ps.name) throw new Error('Invalid Params unexpectedly passed. This is a BUG. Please report it to the development team.'); - const emoji = await this.customEmojiService.getEmojiByName(ps.name); + if (!nameNfc) throw new Error('Invalid Params unexpectedly passed. This is a BUG. Please report it to the development team.'); + const emoji = await this.customEmojiService.getEmojiByName(nameNfc); if (!emoji) throw new ApiError(meta.errors.noSuchEmoji); emojiId = emoji.id; } await this.customEmojiService.update(emojiId, { driveFile, - name: ps.name, - category: ps.category, - aliases: ps.aliases, + name: nameNfc, + category: ps.category?.normalize('NFC'), + aliases: ps.aliases?.map(a => a.normalize('NFC')), license: ps.license, isSensitive: ps.isSensitive, localOnly: ps.localOnly, diff --git a/packages/frontend/src/components/MkAutocomplete.vue b/packages/frontend/src/components/MkAutocomplete.vue index 8b665bfacd..b04a1c92e7 100644 --- a/packages/frontend/src/components/MkAutocomplete.vue +++ b/packages/frontend/src/components/MkAutocomplete.vue @@ -238,7 +238,7 @@ function exec() { return; } - emojis.value = searchEmoji(props.q.toLowerCase(), emojiDb.value); + emojis.value = searchEmoji(props.q.normalize('NFC').toLowerCase(), emojiDb.value); } else if (props.type === 'mfmTag') { if (!props.q || props.q === '') { mfmTags.value = MFM_TAGS; diff --git a/packages/frontend/src/components/MkEmojiPicker.vue b/packages/frontend/src/components/MkEmojiPicker.vue index 1219a29d85..53fca97fc0 100644 --- a/packages/frontend/src/components/MkEmojiPicker.vue +++ b/packages/frontend/src/components/MkEmojiPicker.vue @@ -205,7 +205,7 @@ watch(q, () => { return; } - const newQ = q.value.replace(/:/g, '').toLowerCase(); + const newQ = q.value.replace(/:/g, '').normalize('NFC').toLowerCase(); const searchCustom = () => { const max = 100; diff --git a/packages/frontend/src/pages/emoji-edit-dialog.vue b/packages/frontend/src/pages/emoji-edit-dialog.vue index 64960fd063..d03d3d9128 100644 --- a/packages/frontend/src/pages/emoji-edit-dialog.vue +++ b/packages/frontend/src/pages/emoji-edit-dialog.vue @@ -33,7 +33,7 @@ SPDX-License-Identifier: AGPL-3.0-only {{ i18n.ts.selectFile }} - + diff --git a/packages/frontend/src/scripts/autocomplete.ts b/packages/frontend/src/scripts/autocomplete.ts index 9fc8f7843e..d942402ffc 100644 --- a/packages/frontend/src/scripts/autocomplete.ts +++ b/packages/frontend/src/scripts/autocomplete.ts @@ -99,7 +99,7 @@ export class Autocomplete { const isHashtag = hashtagIndex !== -1; const isMfmParam = mfmParamIndex !== -1 && afterLastMfmParam?.includes('.') && !afterLastMfmParam?.includes(' '); const isMfmTag = mfmTagIndex !== -1 && !isMfmParam; - const isEmoji = emojiIndex !== -1 && text.split(/:[a-z0-9_+\-]+:/).pop()!.includes(':'); + const isEmoji = emojiIndex !== -1 && text.split(/:[\p{Letter}\p{Number}\p{Mark}_+-]+:/u).pop()!.includes(':'); let opened = false; @@ -125,7 +125,7 @@ export class Autocomplete { if (isEmoji && !opened && this.onlyType.includes('emoji')) { const emoji = text.substring(emojiIndex + 1); if (!emoji.includes(' ')) { - this.open('emoji', emoji); + this.open('emoji', emoji.normalize('NFC')); opened = true; } }