vim-jpの全てのmessageを取得
2021-07-08
2021-07-02
00:35:48
channelのmesages pageに日付タグをつけるのは良さげ
同じ日に別のchannelで話されていることを見つけられる
既知の問題
メッセージの投稿順が逆だこれtakker.icon
sort()で逆順にしなきゃ
実装したいこと
ページめくり用リンクを貼っておきたい
code:sh
code:script.ts
import {getList, getMessages} from './mod.ts';
import {getUsers} from '../vim-jpのuser情報をscrapbox_json_dataに変換/mod.ts';
import {getChannels} from '../vim-jpのchannel情報をscrapbox_json_dataに変換/mod.ts';
const list = (await getList());
let counter = 0;
const messages = (await throttle(list.map(({id, date, url}) => async () => {
const index = ++counter;
console.log([${index}/${list.length}] Start fetching...);
const messages = await getMessages(id, date, url);
console.log([${index}/${list.length}] Fetched.);
return messages;
}), {maxInProgress: 100}))
.map(({messages, ...rest}) => ({
messages: messages.filter(({subtype}) => !subtype?.includes?.('channel')),
...rest,
}))
.filter(({messages}) => messages.length > 0);
// scrapbox json dataに変換する
const users = Object.fromEntries((await getUsers()).map(({id, name}) => id, name)); const channels = await getChannels();
const messagePages = messages.map(({id, date, messages}) => {
const channel = channelsid; const title = ${channel} ${date};
const updated = Math.round(Math.max(...messages.map(message => parseFloat(message.ts))));
const created = Math.round(Math.min(...messages.map(message => parseFloat(message.ts))));
return {
title,
updated,
created,
lines: [
title,
...messages.flatMap(message => {
const lines = message.text?.split?.('\n') ?? [];
const reactions = message.reactions?.map?.(reaction => {
const emoji = [/takker-vim-jp-emoji/${reaction.name}.icon];
const userIcons = reaction.users.map(user => [${users[user]}.icon]).join('');
const count = reaction.count - reaction.users.length;
return ${emoji}${userIcons}${count > 0 ? +${count} : ''};
}) ?? [];
return [
[${username}.icon]${lines[0]},
...lines.slice(1).map(text => ${text}),
...reactions,
];
}),
'',
#${date},
#${channel}
],
}
});
const pages = messagePages;
// 2000件づつに分けて格納する
const size = 2000;
const maxChunk = Math.floor(pages.length / size) + 1;
for (let i = 0; i < maxChunk; i++) {
console.log([${i}/${maxChunk}] Writing a json file...);
await Deno.writeTextFile(
messages-${i}.json,
JSON.stringify({pages: pages.slice(i * size, (i + 1) * size)})
);
console.log([${i}/${maxChunk}] Wrote.);
}
messageが格納されているjson fileからmessageを取得する
code:mod.ts
// 認証
const octokit = new Octokit({ auth: Deno.args0 }); export async function getMessages(id: string, date: string, url: string) {
const res = await fetch(url);
const messages = await res.json();
return {date, id, messages};
}
messageが入っているjson fileへのpathを取得する
code:mod.ts
type GitHubResponse = {
data: {
name: string;
path: string;
url: string;
download_url: string | null;
type: 'dir' | 'file';
}[];
};
export async function getList() {
const slackLog: GitHubResponse = await octokit.rest.repos.getContent({
owner: 'vim-jp',
repo: 'slacklog',
ref: 'log-data',
path: 'slacklog_data',
});
const channelDirs = slackLog.data.filter(({type}) => type === 'dir');
const messageURLs = await throttle(channelDirs.map(({name, path}) => async () => {
const messages: GitHubResponse = await octokit.rest.repos.getContent({
owner: 'vim-jp',
repo: 'slacklog',
ref: 'log-data',
path,
});
return messages.data.map(message => ({
id: name,
date: message.name.slice(0, -5),
url: message.download_url
}));
}), {maxInProgress: 10});
return messageURLs.flat();
}
channle名とidとの対応を取得する
code:script.ts
type Channel = {
id: string;
name: string;
};
async function getChannels() {
const channels = (await res.json()) as Channel[];
return Object.fromEntries(channels.map(({id, name}) => id, name)); }