This commit is contained in:
syuilo 2018-10-21 09:20:11 +09:00
parent aa50d0ee11
commit 969b6dbcad
No known key found for this signature in database
GPG Key ID: BDC4C49D06AB9D69
14 changed files with 584 additions and 420 deletions

View File

@ -112,12 +112,42 @@ export default Vue.extend({
}
},
created() {
(this as any).api('chart', {
limit: 35
}).then(chart => {
this.chart = chart;
async created() {
const limit = 35;
const [perHour, perDay] = await Promise.all([Promise.all([
(this as any).api('charts/users', { limit: limit, span: 'hour' }),
(this as any).api('charts/notes', { limit: limit, span: 'hour' }),
(this as any).api('charts/drive', { limit: limit, span: 'hour' }),
(this as any).api('charts/network', { limit: limit, span: 'hour' })
]), Promise.all([
(this as any).api('charts/users', { limit: limit, span: 'day' }),
(this as any).api('charts/notes', { limit: limit, span: 'day' }),
(this as any).api('charts/drive', { limit: limit, span: 'day' }),
(this as any).api('charts/network', { limit: limit, span: 'day' })
])]);
const chart = {
perHour: [],
perDay: []
};
for (let i = 0; i < limit; i++) {
chart.perHour.push({
users: perHour[0][i],
notes: perHour[1][i],
drive: perHour[2][i],
network: perHour[3][i]
});
chart.perDay.push({
users: perDay[0][i],
notes: perDay[1][i],
drive: perDay[2][i],
network: perDay[3][i]
});
}
this.chart = chart;
},
methods: {
@ -586,7 +616,7 @@ export default Vue.extend({
borderWidth: 2,
pointBackgroundColor: '#fff',
lineTension: 0,
data: data.map(x => ({ t: x.date, y: x.incomingRequests }))
data: data.map(x => ({ t: x.date, y: x.incoming }))
}]
}];
},
@ -594,7 +624,7 @@ export default Vue.extend({
networkTimeChart(): any {
const data = this.stats.slice().reverse().map(x => ({
date: new Date(x.date),
time: x.network.requests != 0 ? (x.network.totalTime / x.network.requests) : 0,
time: x.network.incomingRequests != 0 ? (x.network.totalTime / x.network.incomingRequests) : 0,
}));
return [{

View File

@ -10,7 +10,7 @@ import { isCollectionOrOrderedCollection, isCollection, IPerson } from '../type'
import { IDriveFile } from '../../../models/drive-file';
import Meta from '../../../models/meta';
import htmlToMFM from '../../../mfm/html-to-mfm';
import { coreChart } from '../../../services/stats';
import { usersChart } from '../../../services/stats';
import { URL } from 'url';
import { resolveNote } from './note';
@ -180,7 +180,7 @@ export async function createPerson(uri: string, resolver?: Resolver): Promise<IU
}
}, { upsert: true });
coreChart.updateUserStats(user, true);
usersChart.update(user, true);
//#endregion
//#region アイコンとヘッダー画像をフェッチ

View File

@ -1,33 +0,0 @@
import $ from 'cafy';
import getParams from '../get-params';
import { coreChart } from '../../../services/stats';
export const meta = {
desc: {
'ja-JP': 'インスタンスの統計を取得します。'
},
params: {
limit: $.num.optional.range(1, 100).note({
default: 30,
desc: {
'ja-JP': '最大数'
}
}),
}
};
export default (params: any) => new Promise(async (res, rej) => {
const [ps, psErr] = getParams(meta, params);
if (psErr) throw psErr;
const [statsPerDay, statsPerHour] = await Promise.all([
coreChart.getStats('day', ps.limit),
coreChart.getStats('hour', ps.limit)
]);
res({
perDay: statsPerDay,
perHour: statsPerHour
});
});

View File

@ -0,0 +1,33 @@
import $ from 'cafy';
import getParams from '../../get-params';
import { driveChart } from '../../../../services/stats';
export const meta = {
desc: {
'ja-JP': 'ドライブの統計を取得します。'
},
params: {
span: $.str.or(['day', 'hour']).note({
desc: {
'ja-JP': '集計のスパン'
}
}),
limit: $.num.optional.range(1, 100).note({
default: 30,
desc: {
'ja-JP': '最大数。例えば 30 を指定したとすると、スパンが"day"の場合は30日分のデータが、スパンが"hour"の場合は30時間分のデータが返ります。'
}
}),
}
};
export default (params: any) => new Promise(async (res, rej) => {
const [ps, psErr] = getParams(meta, params);
if (psErr) throw psErr;
const stats = await driveChart.getStats(ps.span as any, ps.limit);
res(stats);
});

View File

@ -0,0 +1,33 @@
import $ from 'cafy';
import getParams from '../../get-params';
import { networkChart } from '../../../../services/stats';
export const meta = {
desc: {
'ja-JP': 'ネットワークの統計を取得します。'
},
params: {
span: $.str.or(['day', 'hour']).note({
desc: {
'ja-JP': '集計のスパン'
}
}),
limit: $.num.optional.range(1, 100).note({
default: 30,
desc: {
'ja-JP': '最大数。例えば 30 を指定したとすると、スパンが"day"の場合は30日分のデータが、スパンが"hour"の場合は30時間分のデータが返ります。'
}
}),
}
};
export default (params: any) => new Promise(async (res, rej) => {
const [ps, psErr] = getParams(meta, params);
if (psErr) throw psErr;
const stats = await networkChart.getStats(ps.span as any, ps.limit);
res(stats);
});

View File

@ -0,0 +1,33 @@
import $ from 'cafy';
import getParams from '../../get-params';
import { notesChart } from '../../../../services/stats';
export const meta = {
desc: {
'ja-JP': '投稿の統計を取得します。'
},
params: {
span: $.str.or(['day', 'hour']).note({
desc: {
'ja-JP': '集計のスパン'
}
}),
limit: $.num.optional.range(1, 100).note({
default: 30,
desc: {
'ja-JP': '最大数。例えば 30 を指定したとすると、スパンが"day"の場合は30日分のデータが、スパンが"hour"の場合は30時間分のデータが返ります。'
}
}),
}
};
export default (params: any) => new Promise(async (res, rej) => {
const [ps, psErr] = getParams(meta, params);
if (psErr) throw psErr;
const stats = await notesChart.getStats(ps.span as any, ps.limit);
res(stats);
});

View File

@ -0,0 +1,33 @@
import $ from 'cafy';
import getParams from '../../get-params';
import { usersChart } from '../../../../services/stats';
export const meta = {
desc: {
'ja-JP': 'ユーザーの統計を取得します。'
},
params: {
span: $.str.or(['day', 'hour']).note({
desc: {
'ja-JP': '集計のスパン'
}
}),
limit: $.num.optional.range(1, 100).note({
default: 30,
desc: {
'ja-JP': '最大数。例えば 30 を指定したとすると、スパンが"day"の場合は30日分のデータが、スパンが"hour"の場合は30時間分のデータが返ります。'
}
}),
}
};
export default (params: any) => new Promise(async (res, rej) => {
const [ps, psErr] = getParams(meta, params);
if (psErr) throw psErr;
const stats = await usersChart.getStats(ps.span as any, ps.limit);
res(stats);
});

View File

@ -7,7 +7,7 @@ import generateUserToken from '../common/generate-native-user-token';
import config from '../../../config';
import Meta from '../../../models/meta';
import RegistrationTicket from '../../../models/registration-tickets';
import { coreChart } from '../../../services/stats';
import { usersChart } from '../../../services/stats';
if (config.recaptcha) {
recaptcha.init({
@ -130,7 +130,7 @@ export default async (ctx: Koa.Context) => {
}, { upsert: true });
//#endregion
coreChart.updateUserStats(account, true);
usersChart.update(account, true);
const res = await pack(account, account, {
detail: true,

View File

@ -17,7 +17,7 @@ const requestStats = require('request-stats');
import activityPub from './activitypub';
import webFinger from './webfinger';
import config from '../config';
import { coreChart } from '../services/stats';
import { networkChart } from '../services/stats';
import apiServer from './api';
// Init app
@ -104,7 +104,7 @@ export default () => new Promise(resolve => {
const outgoingBytes = queue.reduce((a, b) => a + b.res.bytes, 0);
queue = [];
coreChart.updateNetworkStats(requests, time, incomingBytes, outgoingBytes);
networkChart.update(requests, time, incomingBytes, outgoingBytes);
}, 5000);
//#endregion
});

View File

@ -17,7 +17,7 @@ import { isLocalUser, IUser, IRemoteUser } from '../../models/user';
import delFile from './delete-file';
import config from '../../config';
import { getDriveFileThumbnailBucket } from '../../models/drive-file-thumbnail';
import { coreChart } from '../stats';
import { driveChart } from '../stats';
const log = debug('misskey:drive:add-file');
@ -389,7 +389,7 @@ export default async function(
});
// 統計を更新
coreChart.updateDriveStats(driveFile, true);
driveChart.update(driveFile, true);
return driveFile;
}

View File

@ -2,7 +2,7 @@ import * as Minio from 'minio';
import DriveFile, { DriveFileChunk, IDriveFile } from '../../models/drive-file';
import DriveFileThumbnail, { DriveFileThumbnailChunk } from '../../models/drive-file-thumbnail';
import config from '../../config';
import { coreChart } from '../stats';
import { driveChart } from '../stats';
export default async function(file: IDriveFile, isExpired = false) {
if (file.metadata.storage == 'minio') {
@ -48,5 +48,5 @@ export default async function(file: IDriveFile, isExpired = false) {
//#endregion
// 統計を更新
coreChart.updateDriveStats(file, false);
driveChart.update(file, false);
}

View File

@ -23,7 +23,7 @@ import registerHashtag from '../register-hashtag';
import isQuote from '../../misc/is-quote';
import { TextElementMention } from '../../mfm/parse/elements/mention';
import { TextElementHashtag } from '../../mfm/parse/elements/hashtag';
import { coreChart } from '../stats';
import { notesChart } from '../stats';
import { erase, unique } from '../../prelude/array';
import insertNoteUnread from './unread';
@ -165,7 +165,7 @@ export default async (user: IUser, data: Option, silent = false) => new Promise<
}
// 統計を更新
coreChart.updateNoteStats(note, true);
notesChart.update(note, true);
// ハッシュタグ登録
tags.map(tag => registerHashtag(user, tag));

View File

@ -6,7 +6,7 @@ import pack from '../../remote/activitypub/renderer';
import { deliver } from '../../queue';
import Following from '../../models/following';
import renderTombstone from '../../remote/activitypub/renderer/tombstone';
import { coreChart } from '../stats';
import { notesChart } from '../stats';
import config from '../../config';
import NoteUnread from '../../models/note-unread';
import read from './read';
@ -63,5 +63,5 @@ export default async function(user: IUser, note: INote) {
//#endregion
// 統計を更新
coreChart.updateNoteStats(note, false);
notesChart.update(note, false);
}

View File

@ -16,6 +16,11 @@ type Span = 'day' | 'hour';
type ChartDocument<T extends Obj> = {
_id: mongo.ObjectID;
/**
*
*/
group?: any;
/**
*
*/
@ -40,6 +45,7 @@ abstract class Chart<T> {
constructor(dbCollectionName: string) {
this.collection = db.get<ChartDocument<T>>(dbCollectionName);
this.collection.createIndex({ span: -1, date: -1 }, { unique: true });
this.collection.createIndex('group');
}
protected async getCurrentStats(span: Span, group?: Obj): Promise<ChartDocument<T>> {
@ -55,10 +61,11 @@ abstract class Chart<T> {
null;
// 現在(今日または今のHour)の統計
const currentStats = await this.collection.findOne(Object.assign({}, {
const currentStats = await this.collection.findOne({
group: group,
span: span,
date: current
}, group));
});
if (currentStats) {
return currentStats;
@ -69,9 +76,10 @@ abstract class Chart<T> {
// * 昨日何もチャートを更新するような出来事がなかった場合は、
// * 統計がそもそも作られずドキュメントが存在しないということがあり得るため、
// * 「昨日の」と決め打ちせずに「もっとも最近の」とします
const mostRecentStats = await this.collection.findOne(Object.assign({}, {
const mostRecentStats = await this.collection.findOne({
group: group,
span: span
}, group), {
}, {
sort: {
date: -1
}
@ -81,11 +89,12 @@ abstract class Chart<T> {
// 現在の統計を初期挿入
const data = this.generateEmptyStats(mostRecentStats.data);
const stats = await this.collection.insert(Object.assign({}, {
const stats = await this.collection.insert({
group: group,
span: span,
date: current,
data: data
}, group));
});
return stats;
} else {
@ -95,18 +104,19 @@ abstract class Chart<T> {
// 空の統計を作成
const data = this.generateInitialStats();
const stats = await this.collection.insert(Object.assign({}, {
const stats = await this.collection.insert({
group: group,
span: span,
date: current,
data: data
}, group));
});
return stats;
}
}
}
protected update(inc: Partial<T>, group?: Obj): void {
protected inc(inc: Partial<T>, group?: Obj): void {
const query: Obj = {};
const dive = (path: string, x: Obj) => {
@ -151,12 +161,13 @@ abstract class Chart<T> {
span == 'day' ? new Date(y, m, d - range) :
span == 'hour' ? new Date(y, m, d, h - range) : null;
const stats = await this.collection.find(Object.assign({
const stats = await this.collection.find({
group: group,
span: span,
date: {
$gt: gt
}
}, group), {
}, {
sort: {
date: -1
},
@ -189,11 +200,11 @@ abstract class Chart<T> {
}
}
type CoreStats = {
/**
//#region Users stats
/**
*
*/
users: {
type UsersStats = {
local: {
/**
* ()
@ -227,12 +238,67 @@ type CoreStats = {
*/
dec: number;
};
};
};
/**
class UsersChart extends Chart<UsersStats> {
constructor() {
super('usersStats');
}
protected generateInitialStats(): UsersStats {
return {
local: {
total: 0,
inc: 0,
dec: 0
},
remote: {
total: 0,
inc: 0,
dec: 0
}
};
}
protected generateEmptyStats(mostRecentStats: UsersStats): UsersStats {
return {
local: {
total: mostRecentStats.local.total,
inc: 0,
dec: 0
},
remote: {
total: mostRecentStats.remote.total,
inc: 0,
dec: 0
}
};
}
public async update(user: IUser, isAdditional: boolean) {
const update: Obj = {};
update.total = isAdditional ? 1 : -1;
if (isAdditional) {
update.inc = 1;
} else {
update.dec = 1;
}
await this.inc({
[isLocalUser(user) ? 'local' : 'remote']: update
});
}
}
export const usersChart = new UsersChart();
//#endregion
//#region Notes stats
/**
* 稿
*/
notes: {
type NotesStats = {
local: {
/**
* 稿 ()
@ -300,12 +366,96 @@ type CoreStats = {
renote: number;
};
};
};
};
/**
* ()
class NotesChart extends Chart<NotesStats> {
constructor() {
super('notesStats');
}
protected generateInitialStats(): NotesStats {
return {
local: {
total: 0,
inc: 0,
dec: 0,
diffs: {
normal: 0,
reply: 0,
renote: 0
}
},
remote: {
total: 0,
inc: 0,
dec: 0,
diffs: {
normal: 0,
reply: 0,
renote: 0
}
}
};
}
protected generateEmptyStats(mostRecentStats: NotesStats): NotesStats {
return {
local: {
total: mostRecentStats.local.total,
inc: 0,
dec: 0,
diffs: {
normal: 0,
reply: 0,
renote: 0
}
},
remote: {
total: mostRecentStats.remote.total,
inc: 0,
dec: 0,
diffs: {
normal: 0,
reply: 0,
renote: 0
}
}
};
}
public async update(note: INote, isAdditional: boolean) {
const update: Obj = {};
update.total = isAdditional ? 1 : -1;
if (isAdditional) {
update.inc = 1;
} else {
update.dec = 1;
}
if (note.replyId != null) {
update.diffs.reply = isAdditional ? 1 : -1;
} else if (note.renoteId != null) {
update.diffs.renote = isAdditional ? 1 : -1;
} else {
update.diffs.normal = isAdditional ? 1 : -1;
}
await this.inc({
[isLocalUser(note._user) ? 'local' : 'remote']: update
});
}
}
export const notesChart = new NotesChart();
//#endregion
//#region Drive stats
/**
*
*/
drive: {
type DriveStats = {
local: {
/**
* ()
@ -369,12 +519,82 @@ type CoreStats = {
*/
decSize: number;
};
};
};
/**
class DriveChart extends Chart<DriveStats> {
constructor() {
super('driveStats');
}
protected generateInitialStats(): DriveStats {
return {
local: {
totalCount: 0,
totalSize: 0,
incCount: 0,
incSize: 0,
decCount: 0,
decSize: 0
},
remote: {
totalCount: 0,
totalSize: 0,
incCount: 0,
incSize: 0,
decCount: 0,
decSize: 0
}
};
}
protected generateEmptyStats(mostRecentStats: DriveStats): DriveStats {
return {
local: {
totalCount: mostRecentStats.local.totalCount,
totalSize: mostRecentStats.local.totalSize,
incCount: 0,
incSize: 0,
decCount: 0,
decSize: 0
},
remote: {
totalCount: mostRecentStats.remote.totalCount,
totalSize: mostRecentStats.remote.totalSize,
incCount: 0,
incSize: 0,
decCount: 0,
decSize: 0
}
};
}
public async update(file: IDriveFile, isAdditional: boolean) {
const update: Obj = {};
update.totalCount = isAdditional ? 1 : -1;
update.totalSize = isAdditional ? file.length : -file.length;
if (isAdditional) {
update.incCount = 1;
update.incSize = file.length;
} else {
update.decCount = 1;
update.decSize = file.length;
}
await this.inc({
[isLocalUser(file.metadata._user) ? 'local' : 'remote']: update
});
}
}
export const driveChart = new DriveChart();
//#endregion
//#region Network stats
/**
*
*/
network: {
type NetworkStats = {
/**
*
*/
@ -400,229 +620,44 @@ type CoreStats = {
*
*/
outgoingBytes: number;
};
};
class CoreChart extends Chart<CoreStats> {
class NetworkChart extends Chart<NetworkStats> {
constructor() {
super('coreStats');
super('networkStats');
}
protected generateInitialStats(): CoreStats {
protected generateInitialStats(): NetworkStats {
return {
users: {
local: {
total: 0,
inc: 0,
dec: 0
},
remote: {
total: 0,
inc: 0,
dec: 0
}
},
notes: {
local: {
total: 0,
inc: 0,
dec: 0,
diffs: {
normal: 0,
reply: 0,
renote: 0
}
},
remote: {
total: 0,
inc: 0,
dec: 0,
diffs: {
normal: 0,
reply: 0,
renote: 0
}
}
},
drive: {
local: {
totalCount: 0,
totalSize: 0,
incCount: 0,
incSize: 0,
decCount: 0,
decSize: 0
},
remote: {
totalCount: 0,
totalSize: 0,
incCount: 0,
incSize: 0,
decCount: 0,
decSize: 0
}
},
network: {
incomingRequests: 0,
outgoingRequests: 0,
totalTime: 0,
incomingBytes: 0,
outgoingBytes: 0
}
};
}
protected generateEmptyStats(mostRecentStats: CoreStats): CoreStats {
protected generateEmptyStats(mostRecentStats: NetworkStats): NetworkStats {
return {
users: {
local: {
total: mostRecentStats.users.local.total,
inc: 0,
dec: 0
},
remote: {
total: mostRecentStats.users.remote.total,
inc: 0,
dec: 0
}
},
notes: {
local: {
total: mostRecentStats.notes.local.total,
inc: 0,
dec: 0,
diffs: {
normal: 0,
reply: 0,
renote: 0
}
},
remote: {
total: mostRecentStats.notes.remote.total,
inc: 0,
dec: 0,
diffs: {
normal: 0,
reply: 0,
renote: 0
}
}
},
drive: {
local: {
totalCount: mostRecentStats.drive.local.totalCount,
totalSize: mostRecentStats.drive.local.totalSize,
incCount: 0,
incSize: 0,
decCount: 0,
decSize: 0
},
remote: {
totalCount: mostRecentStats.drive.remote.totalCount,
totalSize: mostRecentStats.drive.remote.totalSize,
incCount: 0,
incSize: 0,
decCount: 0,
decSize: 0
}
},
network: {
incomingRequests: 0,
outgoingRequests: 0,
totalTime: 0,
incomingBytes: 0,
outgoingBytes: 0
}
};
}
public async updateUserStats(user: IUser, isAdditional: boolean) {
const origin = isLocalUser(user) ? 'local' : 'remote';
const update: Obj = {};
update.total = isAdditional ? 1 : -1;
if (isAdditional) {
update.inc = 1;
} else {
update.dec = 1;
}
const inc: Obj = {
users: {}
};
inc.users[origin] = update;
await this.update(inc);
}
public async updateNoteStats(note: INote, isAdditional: boolean) {
const origin = isLocalUser(note._user) ? 'local' : 'remote';
const update: Obj = {};
update.total = isAdditional ? 1 : -1;
if (isAdditional) {
update.inc = 1;
} else {
update.dec = 1;
}
if (note.replyId != null) {
update.diffs.reply = isAdditional ? 1 : -1;
} else if (note.renoteId != null) {
update.diffs.renote = isAdditional ? 1 : -1;
} else {
update.diffs.normal = isAdditional ? 1 : -1;
}
const inc: Obj = {
notes: {}
};
inc.notes[origin] = update;
await this.update(inc);
}
public async updateDriveStats(file: IDriveFile, isAdditional: boolean) {
const origin = isLocalUser(file.metadata._user) ? 'local' : 'remote';
const update: Obj = {};
update.totalCount = isAdditional ? 1 : -1;
update.totalSize = isAdditional ? file.length : -file.length;
if (isAdditional) {
update.incCount = 1;
update.incSize = file.length;
} else {
update.decCount = 1;
update.decSize = file.length;
}
const inc: Obj = {
drive: {}
};
inc.drive[origin] = update;
await this.update(inc);
}
public async updateNetworkStats(incomingRequests: number, time: number, incomingBytes: number, outgoingBytes: number) {
const inc: Partial<CoreStats> = {
network: {
public async update(incomingRequests: number, time: number, incomingBytes: number, outgoingBytes: number) {
const inc: Partial<NetworkStats> = {
incomingRequests: incomingRequests,
totalTime: time,
incomingBytes: incomingBytes,
outgoingBytes: outgoingBytes
}
};
await this.update(inc);
await this.inc(inc);
}
}
export const coreChart = new CoreChart();
export const networkChart = new NetworkChart();
//#endregion