From ee5720df2c93a852d5429cb3919cdecc8b8051f8 Mon Sep 17 00:00:00 2001 From: MeiMei <30769358+mei23@users.noreply.github.com> Date: Thu, 25 Apr 2019 04:07:39 +0900 Subject: [PATCH 1/7] Fix #4704 (#4797) * Fix #4632 * Fix #4795 --- src/remote/activitypub/models/note.ts | 2 +- src/server/api/endpoints/ap/show.ts | 28 ++++++++++++++++++++++++++- src/services/note/create.ts | 4 +++- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/remote/activitypub/models/note.ts b/src/remote/activitypub/models/note.ts index 850b5e65e6..d7ca625521 100644 --- a/src/remote/activitypub/models/note.ts +++ b/src/remote/activitypub/models/note.ts @@ -247,7 +247,7 @@ export async function resolveNote(value: string | IObject, resolver?: Resolver): // リモートサーバーからフェッチしてきて登録 // ここでuriの代わりに添付されてきたNote Objectが指定されていると、サーバーフェッチを経ずにノートが生成されるが // 添付されてきたNote Objectは偽装されている可能性があるため、常にuriを指定してサーバーフェッチを行う。 - return await createNote(uri, resolver).catch(e => { + return await createNote(uri, resolver, true).catch(e => { if (e.name === 'duplicated') { return fetchNote(uri).then(note => { if (note == null) { diff --git a/src/server/api/endpoints/ap/show.ts b/src/server/api/endpoints/ap/show.ts index 1bb15117dd..9724a044b1 100644 --- a/src/server/api/endpoints/ap/show.ts +++ b/src/server/api/endpoints/ap/show.ts @@ -101,6 +101,32 @@ async function fetchAny(uri: string) { // /@user のような正規id以外で取得できるURIが指定されていた場合、ここで初めて正規URIが確定する // これはDBに存在する可能性があるため再度DB検索 if (uri !== object.id) { + if (object.id.startsWith(config.url + '/')) { + const parts = object.id.split('/'); + const id = parts.pop(); + const type = parts.pop(); + + if (type === 'notes') { + const note = await Notes.findOne(id); + + if (note) { + return { + type: 'Note', + object: await Notes.pack(note, null, { detail: true }) + }; + } + } else if (type === 'users') { + const user = await Users.findOne(id); + + if (user) { + return { + type: 'User', + object: await Users.pack(user, null, { detail: true }) + }; + } + } + } + const [user, note] = await Promise.all([ Users.findOne({ uri: object.id }), Notes.findOne({ uri: object.id }) @@ -120,7 +146,7 @@ async function fetchAny(uri: string) { } if (['Note', 'Question', 'Article'].includes(object.type)) { - const note = await createNote(object.id); + const note = await createNote(object.id, undefined, true); return { type: 'Note', object: await Notes.pack(note!, null, { detail: true }) diff --git a/src/services/note/create.ts b/src/services/note/create.ts index ce229d6393..2195ecc55f 100644 --- a/src/services/note/create.ts +++ b/src/services/note/create.ts @@ -240,7 +240,9 @@ export default async (user: User, data: Option, silent = false) => new Promise Date: Thu, 25 Apr 2019 04:17:03 +0900 Subject: [PATCH 2/7] Refactorgin --- src/services/note/create.ts | 139 ++++++++++++++++++------------------ 1 file changed, 68 insertions(+), 71 deletions(-) diff --git a/src/services/note/create.ts b/src/services/note/create.ts index 2195ecc55f..53e77b4ef2 100644 --- a/src/services/note/create.ts +++ b/src/services/note/create.ts @@ -106,8 +106,6 @@ type Option = { }; export default async (user: User, data: Option, silent = false) => new Promise(async (res, rej) => { - const isFirstNote = user.notesCount === 0; - if (data.createdAt == null) data.createdAt = new Date(); if (data.visibility == null) data.visibility = 'public'; if (data.viaMobile == null) data.viaMobile = false; @@ -195,8 +193,6 @@ export default async (user: User, data: Option, silent = false) => new Promise new Promise new Promise { - nm.deliver(); - }); + Promise.all(nmRelatedPromises).then(() => { + nm.deliver(); + }); + } // Register to search database index(note); From b90d473ae5be5c7077c8b0af22ef28e2b4276fbd Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 25 Apr 2019 04:26:58 +0900 Subject: [PATCH 3/7] Fix layout --- src/client/app/mobile/views/components/ui.nav.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/app/mobile/views/components/ui.nav.vue b/src/client/app/mobile/views/components/ui.nav.vue index 1177cab4a0..9a3ade4c63 100644 --- a/src/client/app/mobile/views/components/ui.nav.vue +++ b/src/client/app/mobile/views/components/ui.nav.vue @@ -43,7 +43,7 @@
- +
From 3ef002e14d01cc36f78f93b96fe5cd29136382d6 Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 25 Apr 2019 04:27:34 +0900 Subject: [PATCH 4/7] Fix bug --- src/client/app/common/views/widgets/broadcast.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/app/common/views/widgets/broadcast.vue b/src/client/app/common/views/widgets/broadcast.vue index 7049945d55..6850ff7afc 100644 --- a/src/client/app/common/views/widgets/broadcast.vue +++ b/src/client/app/common/views/widgets/broadcast.vue @@ -18,7 +18,7 @@

{{ $t('fetching') }}

{{ announcements.length == 0 ? $t('no-broadcasts') : announcements[i].title }}

- +

From 772258b0b8eac5ffb0dcae5a387c190d2eecc0e3 Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 25 Apr 2019 04:32:01 +0900 Subject: [PATCH 5/7] Fix #4793 --- src/client/app/mobile/views/pages/user/index.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/app/mobile/views/pages/user/index.vue b/src/client/app/mobile/views/pages/user/index.vue index cc53ae0cfe..7afa06d4a4 100644 --- a/src/client/app/mobile/views/pages/user/index.vue +++ b/src/client/app/mobile/views/pages/user/index.vue @@ -18,7 +18,7 @@

- + {{ $t('follows-you') }}
From 0db54386cdef3f444f1afb4f3b8bfcaeab7ac68d Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 25 Apr 2019 07:46:39 +0900 Subject: [PATCH 6/7] Resolve #3119 --- CHANGELOG.md | 13 ++++ package.json | 3 +- .../app/common/scripts/gen-search-query.ts | 31 ++++++++ src/client/app/common/scripts/search.ts | 2 +- .../app/common/views/components/user-list.vue | 8 +-- .../app/common/views/deck/deck.notes.vue | 8 +-- .../common/views/deck/deck.search-column.vue | 5 +- .../app/desktop/views/components/notes.vue | 6 +- src/client/app/desktop/views/home/search.vue | 5 +- .../app/mobile/views/components/notes.vue | 8 +-- src/client/app/mobile/views/pages/search.vue | 5 +- src/db/elasticsearch.ts | 72 +++++++------------ src/server/api/endpoints/notes/search.ts | 64 ++++++++++++----- src/services/note/create.ts | 7 +- 14 files changed, 147 insertions(+), 90 deletions(-) create mode 100644 src/client/app/common/scripts/gen-search-query.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index e797664c12..f50bcef576 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,19 @@ mongodb: 8. master ブランチに戻す 9. enjoy +11.4.0 (2019/04/25) +------------------- +### Improvements +* 検索でローカルの投稿のみに絞れるように +* 検索で特定のインスタンスの投稿のみに絞れるように +* 検索で特定のユーザーの投稿のみに絞れるように + +### Fixes +* 投稿が増殖する問題を修正 +* ストリームで過去の投稿が流れてくる問題を修正 +* モバイル版のユーザーページで遷移してもユーザー名が変わらない問題を修正 +* お知らせを切り替えても内容が変わらない問題を修正 + 11.3.1 (2019/04/24) ------------------- ### Fixes diff --git a/package.json b/package.json index 00cbdc73c5..56a5ac0dcd 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "format": "gulp format" }, "dependencies": { + "@elastic/elasticsearch": "7.0.0-rc.2", "@fortawesome/fontawesome-svg-core": "1.2.15", "@fortawesome/free-brands-svg-icons": "5.7.2", "@fortawesome/free-regular-svg-icons": "5.7.2", @@ -35,7 +36,6 @@ "@types/dateformat": "3.0.0", "@types/deep-equal": "1.0.1", "@types/double-ended-queue": "2.1.0", - "@types/elasticsearch": "5.0.32", "@types/file-type": "10.9.1", "@types/gulp": "4.0.6", "@types/gulp-mocha": "0.0.32", @@ -113,7 +113,6 @@ "deep-equal": "1.0.1", "diskusage": "1.1.0", "double-ended-queue": "2.1.0-0", - "elasticsearch": "15.4.1", "emojilib": "2.4.0", "eslint": "5.16.0", "eslint-plugin-vue": "5.2.2", diff --git a/src/client/app/common/scripts/gen-search-query.ts b/src/client/app/common/scripts/gen-search-query.ts new file mode 100644 index 0000000000..fc26cb7f78 --- /dev/null +++ b/src/client/app/common/scripts/gen-search-query.ts @@ -0,0 +1,31 @@ +import parseAcct from '../../../../misc/acct/parse'; +import { host as localHost } from '../../config'; + +export async function genSearchQuery(v: any, q: string) { + let host: string; + let userId: string; + if (q.split(' ').some(x => x.startsWith('@'))) { + for (const at of q.split(' ').filter(x => x.startsWith('@')).map(x => x.substr(1))) { + if (at.includes('.')) { + if (at === localHost || at === '.') { + host = null; + } else { + host = at; + } + } else { + const user = await v.$root.api('users/show', parseAcct(at)).catch(x => null); + if (user) { + userId = user.id; + } else { + // todo: show error + } + } + } + + } + return { + query: q.split(' ').filter(x => !x.startsWith('/') && !x.startsWith('@')).join(' '), + host: host, + userId: userId + }; +} diff --git a/src/client/app/common/scripts/search.ts b/src/client/app/common/scripts/search.ts index c44581817b..2897ed6318 100644 --- a/src/client/app/common/scripts/search.ts +++ b/src/client/app/common/scripts/search.ts @@ -3,7 +3,7 @@ import { faHistory } from '@fortawesome/free-solid-svg-icons'; export async function search(v: any, q: string) { q = q.trim(); - if (q.startsWith('@')) { + if (q.startsWith('@') && !q.includes(' ')) { v.$router.push(`/${q}`); return; } diff --git a/src/client/app/common/views/components/user-list.vue b/src/client/app/common/views/components/user-list.vue index b8bcc35d82..53577bad00 100644 --- a/src/client/app/common/views/components/user-list.vue +++ b/src/client/app/common/views/components/user-list.vue @@ -60,9 +60,9 @@ export default Vue.extend({ }, methods: { - init() { + async init() { this.fetching = true; - this.makePromise().then(x => { + await (this.makePromise()).then(x => { if (Array.isArray(x)) { this.us = x; } else { @@ -76,9 +76,9 @@ export default Vue.extend({ }); }, - fetchMoreUsers() { + async fetchMoreUsers() { this.fetchingMoreUsers = true; - this.makePromise(this.cursor).then(x => { + await (this.makePromise(this.cursor)).then(x => { this.us = this.us.concat(x.users); this.cursor = x.cursor; this.fetchingMoreUsers = false; diff --git a/src/client/app/common/views/deck/deck.notes.vue b/src/client/app/common/views/deck/deck.notes.vue index bc67f4911c..680b44bc81 100644 --- a/src/client/app/common/views/deck/deck.notes.vue +++ b/src/client/app/common/views/deck/deck.notes.vue @@ -110,11 +110,11 @@ export default Vue.extend({ this.init(); }, - init() { + async init() { this.queue = []; this.notes = []; this.fetching = true; - this.makePromise().then(x => { + await (this.makePromise()).then(x => { if (Array.isArray(x)) { this.notes = x; } else { @@ -129,10 +129,10 @@ export default Vue.extend({ }); }, - fetchMore() { + async fetchMore() { if (!this.more || this.moreFetching) return; this.moreFetching = true; - this.makePromise(this.notes[this.notes.length - 1].id).then(x => { + await (this.makePromise(this.notes[this.notes.length - 1].id)).then(x => { this.notes = this.notes.concat(x.notes); this.more = x.more; this.moreFetching = false; diff --git a/src/client/app/common/views/deck/deck.search-column.vue b/src/client/app/common/views/deck/deck.search-column.vue index ab19bdaab6..17ee2ef454 100644 --- a/src/client/app/common/views/deck/deck.search-column.vue +++ b/src/client/app/common/views/deck/deck.search-column.vue @@ -14,6 +14,7 @@ import Vue from 'vue'; import XColumn from './deck.column.vue'; import XNotes from './deck.notes.vue'; +import { genSearchQuery } from '../../../common/scripts/gen-search-query'; const limit = 20; @@ -25,10 +26,10 @@ export default Vue.extend({ data() { return { - makePromise: cursor => this.$root.api('notes/search', { + makePromise: async cursor => this.$root.api('notes/search', { limit: limit + 1, offset: cursor ? cursor : undefined, - query: this.q + ...(await genSearchQuery(this, this.q)) }).then(notes => { if (notes.length == limit + 1) { notes.pop(); diff --git a/src/client/app/desktop/views/components/notes.vue b/src/client/app/desktop/views/components/notes.vue index 9044ad3478..87fdc749de 100644 --- a/src/client/app/desktop/views/components/notes.vue +++ b/src/client/app/desktop/views/components/notes.vue @@ -105,9 +105,9 @@ export default Vue.extend({ this.init(); }, - init() { + async init() { this.fetching = true; - this.makePromise().then(x => { + await (this.makePromise()).then(x => { if (Array.isArray(x)) { this.notes = x; } else { @@ -122,7 +122,7 @@ export default Vue.extend({ }); }, - fetchMore() { + async fetchMore() { if (!this.more || this.moreFetching || this.notes.length === 0) return; this.moreFetching = true; this.makePromise(this.notes[this.notes.length - 1].id).then(x => { diff --git a/src/client/app/desktop/views/home/search.vue b/src/client/app/desktop/views/home/search.vue index 84153d18c4..50c6456158 100644 --- a/src/client/app/desktop/views/home/search.vue +++ b/src/client/app/desktop/views/home/search.vue @@ -14,6 +14,7 @@ import Vue from 'vue'; import i18n from '../../../i18n'; import Progress from '../../../common/scripts/loading'; +import { genSearchQuery } from '../../../common/scripts/gen-search-query'; const limit = 20; @@ -21,10 +22,10 @@ export default Vue.extend({ i18n: i18n('desktop/views/pages/search.vue'), data() { return { - makePromise: cursor => this.$root.api('notes/search', { + makePromise: async cursor => this.$root.api('notes/search', { limit: limit + 1, offset: cursor ? cursor : undefined, - query: this.q + ...(await genSearchQuery(this, this.q)) }).then(notes => { if (notes.length == limit + 1) { notes.pop(); diff --git a/src/client/app/mobile/views/components/notes.vue b/src/client/app/mobile/views/components/notes.vue index 2e42300717..5ad80c286d 100644 --- a/src/client/app/mobile/views/components/notes.vue +++ b/src/client/app/mobile/views/components/notes.vue @@ -106,9 +106,9 @@ export default Vue.extend({ this.init(); }, - init() { + async init() { this.fetching = true; - this.makePromise().then(x => { + await (this.makePromise()).then(x => { if (Array.isArray(x)) { this.notes = x; } else { @@ -123,10 +123,10 @@ export default Vue.extend({ }); }, - fetchMore() { + async fetchMore() { if (!this.more || this.moreFetching || this.notes.length === 0) return; this.moreFetching = true; - this.makePromise(this.notes[this.notes.length - 1].id).then(x => { + await (this.makePromise(this.notes[this.notes.length - 1].id)).then(x => { this.notes = this.notes.concat(x.notes); this.more = x.more; this.moreFetching = false; diff --git a/src/client/app/mobile/views/pages/search.vue b/src/client/app/mobile/views/pages/search.vue index 0225dd9e9f..45f3837907 100644 --- a/src/client/app/mobile/views/pages/search.vue +++ b/src/client/app/mobile/views/pages/search.vue @@ -12,6 +12,7 @@ import Vue from 'vue'; import i18n from '../../../i18n'; import Progress from '../../../common/scripts/loading'; +import { genSearchQuery } from '../../../common/scripts/gen-search-query'; const limit = 20; @@ -19,10 +20,10 @@ export default Vue.extend({ i18n: i18n('mobile/views/pages/search.vue'), data() { return { - makePromise: cursor => this.$root.api('notes/search', { + makePromise: async cursor => this.$root.api('notes/search', { limit: limit + 1, untilId: cursor ? cursor : undefined, - query: this.q + ...(await genSearchQuery(this, this.q)) }).then(notes => { if (notes.length == limit + 1) { notes.pop(); diff --git a/src/db/elasticsearch.ts b/src/db/elasticsearch.ts index d54b01763b..02c9e88d9c 100644 --- a/src/db/elasticsearch.ts +++ b/src/db/elasticsearch.ts @@ -1,41 +1,30 @@ -import * as elasticsearch from 'elasticsearch'; +import * as elasticsearch from '@elastic/elasticsearch'; import config from '../config'; -import Logger from '../services/logger'; - -const esLogger = new Logger('es'); const index = { settings: { analysis: { - normalizer: { - lowercase_normalizer: { - type: 'custom', - filter: ['lowercase'] - } - }, analyzer: { - bigram: { - tokenizer: 'bigram_tokenizer' - } - }, - tokenizer: { - bigram_tokenizer: { - type: 'nGram', - min_gram: 2, - max_gram: 2 + ngram: { + tokenizer: 'ngram' } } } }, mappings: { - note: { - properties: { - text: { - type: 'text', - index: true, - analyzer: 'bigram', - normalizer: 'lowercase_normalizer' - } + properties: { + text: { + type: 'text', + index: true, + analyzer: 'ngram', + }, + userId: { + type: 'keyword', + index: true, + }, + userHost: { + type: 'keyword', + index: true, } } } @@ -43,31 +32,20 @@ const index = { // Init ElasticSearch connection const client = config.elasticsearch ? new elasticsearch.Client({ - host: `${config.elasticsearch.host}:${config.elasticsearch.port}` + node: `http://${config.elasticsearch.host}:${config.elasticsearch.port}`, + pingTimeout: 30000 }) : null; if (client) { - // Send a HEAD request - client.ping({ - // Ping usually has a 3000ms timeout - requestTimeout: 30000 - }, error => { - if (error) { - esLogger.error('elasticsearch is down!'); - } else { - esLogger.succ('elasticsearch is available!'); - } - }); - client.indices.exists({ - index: 'misskey' + index: 'misskey_note' }).then(exist => { - if (exist) return; - - client.indices.create({ - index: 'misskey', - body: index - }); + if (!exist.body) { + client.indices.create({ + index: 'misskey_note', + body: index + }); + } }); } diff --git a/src/server/api/endpoints/notes/search.ts b/src/server/api/endpoints/notes/search.ts index daf992b639..65ce20074a 100644 --- a/src/server/api/endpoints/notes/search.ts +++ b/src/server/api/endpoints/notes/search.ts @@ -5,6 +5,7 @@ import { ApiError } from '../../error'; import { Notes } from '../../../../models'; import { In } from 'typeorm'; import { types, bool } from '../../../../misc/schema'; +import { ID } from '../../../../misc/cafy-id'; export const meta = { desc: { @@ -29,7 +30,17 @@ export const meta = { offset: { validator: $.optional.num.min(0), default: 0 - } + }, + + host: { + validator: $.optional.nullable.str, + default: undefined + }, + + userId: { + validator: $.optional.nullable.type(ID), + default: null + }, }, res: { @@ -54,30 +65,51 @@ export const meta = { export default define(meta, async (ps, me) => { if (es == null) throw new ApiError(meta.errors.searchingNotAvailable); - const response = await es.search({ - index: 'misskey', - type: 'note', + const userQuery = ps.userId != null ? [{ + term: { + userId: ps.userId + } + }] : []; + + const hostQuery = ps.userId == null ? + ps.host === null ? [{ + bool: { + must_not: { + exists: { + field: 'userHost' + } + } + } + }] : ps.host !== undefined ? [{ + term: { + userHost: ps.host + } + }] : [] + : []; + + const result = await es.search({ + index: 'misskey_note', body: { size: ps.limit!, from: ps.offset, query: { - simple_query_string: { - fields: ['text'], - query: ps.query, - default_operator: 'and' + bool: { + must: [{ + simple_query_string: { + fields: ['text'], + query: ps.query.toLowerCase(), + default_operator: 'and' + }, + }, ...hostQuery, ...userQuery] } }, - sort: [ - { _doc: 'desc' } - ] + sort: [{ + _doc: 'desc' + }] } }); - if (response.hits.total === 0) { - return []; - } - - const hits = response.hits.hits.map((hit: any) => hit.id); + const hits = result.body.hits.hits.map((hit: any) => hit._id); if (hits.length === 0) return []; diff --git a/src/services/note/create.ts b/src/services/note/create.ts index 53e77b4ef2..dd47632caa 100644 --- a/src/services/note/create.ts +++ b/src/services/note/create.ts @@ -435,11 +435,12 @@ function index(note: Note) { if (note.text == null || config.elasticsearch == null) return; es!.index({ - index: 'misskey', - type: 'note', + index: 'misskey_note', id: note.id.toString(), body: { - text: note.text + text: note.text.toLowerCase(), + userId: note.userId, + userHost: note.userHost } }); } From fe87d16d463f43504388aec178acced71b9a431e Mon Sep 17 00:00:00 2001 From: syuilo Date: Thu, 25 Apr 2019 07:48:12 +0900 Subject: [PATCH 7/7] 11.4.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 56a5ac0dcd..b1900c8fa7 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "misskey", "author": "syuilo ", - "version": "11.3.1", + "version": "11.4.0", "codename": "daybreak", "repository": { "type": "git",