scrapbox-cache-fetch
用途
fetchの数を減らして待ち時間を減らす
scrapbox.ioへのサーバー負荷を減らす
挙動
初回 or cacheが存在しない
APIを叩く
cacheの取得日時からmaxAge秒過ぎている
対象projectの最終更新日時がcacheの取得日時よりあとの場合のみAPIを叩く
特に更新がなければcacheを返す
otherwise
cacheを返す
interface
getAllLinks(projects, {maxAge, verbose})
projecsのすべてのリンクを取得する
maxAge (optional)
cacheの有効期間
単位は秒
default: 3600 * 24
verbose (optional)
ログを出力する
default: false
reload (optional)
networkから取得し直す
default: false
返り値の型
code:ts
type LinksResult = {
project: string;
results: {
title: string;
links: string[];
}[];
type: 'network' : 'cache';
};
getAllIcons(project, {maxAge})
projectのすべてのアイコン入りページを取得する
parametersはgetAllLinksと同じ
返り値の型
code:ts
type IconsResult = {
project: string;
results: string[];
type: 'network' : 'cache';
};
getAllPageCards(projects, {maxAge})
parametersはgetAllLinksと同じ
返り値の型
code:ts
type CardsResult = {
project: string;
results: {
title: string;
image: string;
descriptions: string[];
}[];
type: 'network' | 'cache';
};
更新通知event
scrapbox-user-script-cache-update-links
e.detail: (LinksResult & {project: stirng})[]
scrapbox-user-script-cache-update-icons
e.detail: (IconsResult & {project: stirng})[]
scrapbox-user-script-cache-update-cards
e.detail: (CardsResult & {project: stirng})[]
window.addEventListener()で受け取れる
既知の問題
実装
code:script.js
export async function getAllLinks(project, {maxAge = 24 * 3600, verbose = false, reload = false} = {}) {
return await getData(
'scrapbox-user-script-cache-update-links',
async () => (await fetchAllLinks(project, verbose))
.map(({title, links}) => {return {title, links}}),
project,
{maxAge, verbose, reload},);
}
export async function getAllIcons(project, {maxAge = 24 * 3600, verbose = false, reload = false} = {}) {
return await getData(
'scrapbox-user-script-cache-update-icons',
() => fetchAllIcons(project, verbose),
project,
{maxAge, verbose, reload},);
}
共通する処理
code:script.js
async function getData(eventName, callback, project, {maxAge, verbose, reload}) {
const lastUpdated = getTimestamp(eventName, project);
const expired = lastUpdated + maxAge;
const now = (new Date()).getTime() / 1000;
if (reload) return await getFromNetwork(eventName, callback, project, verbose);
if (expired < now) {
const updated = await getProjectUpdated(project, verbose);
_log('getData', verbose, {lastUpdated, expired, now, updated});
if (lastUpdated < updated) {
return await getFromNetwork(eventName, callback, project, verbose);
}
}
const cache = getFromCache(eventName, project, verbose);
if (cache.results) return cache;
return await getFromNetwork(eventName, callback, project, verbose);
}
Local Storageと通信を行う
code:script.js
function getTimestamp(eventName, project) {
return parseInt(localStorage.getItem(${eventName}-${project}-timestamp) ?? 0);
}
function getFromCache(eventName, project, verbose) {
_log('getFromCache', verbose, Get data of /${project} from cache.);
const dataString = localStorage.getItem(${eventName}-${project});
return {
project,
results: dataString !== null ? JSON.parse(dataString) : undefined,
state: 'cache',
};
}
async function getFromNetwork(eventName, callback, project, verbose) {
_log('getFromNetwork', verbose, Get data of /${project} from network.);
const results = await callback();
UpdateCache(eventName, project, results, verbose);
return {project, results, state: 'network',};
}
function UpdateCache(eventName, project, results, verbose) {
localStorage.setItem(${eventName}-${project}, JSON.stringify(results));
localStorage.setItem(${eventName}-${project}-timestamp, Math.round((new Date()).getTime() / 1000));
// 同じタブのscriptに更新通知を出す
dispatchUpdateEvent(eventName, project, {state: 'cache', verbose});
}
function dispatchUpdateEvent(eventName, project, {state, verbose}) {
const {results} = getFromCache(eventName, project, verbose);
window.dispatchEvent(new CustomEvent(eventName, {bubbles: true, detail: {
project, results, state,
}}));
}
実際にAPIを叩く関数
code:script.js
async function fetchAllLinks(project, verbose) {
let followingId = null;
const linksData = [];
const promises = []; //処理待ち用
_log('fetchAllLinks', verbose, Start loading links from /${project}...);
do {
_log('fetchAllLinks', verbose, Loading links from /${project}: followingId = ${followingId});
const response = await (!followingId ?
fetch(/api/pages/${project}/search/titles) :
fetch(/api/pages/${project}/search/titles?followingId=${followingId})
)
followingId = response.headers.get('X-Following-Id');
promises.push(response.json().then(links => linksData.push(...links)));
} while(followingId);
// 全てのリンク情報を配列に格納し終わったら返す
await Promise.all(promises);
_log('fetchAllLinks', verbose, Loaded all links from /${project}: , linksData);
return linksData;
}
code:script.js
async function fetchAllIcons(project, verbose) {
const cards = await fetchAllCards(project, verbose);
return cards.flatMap(({title, image}) => image !== null ? title : []); }
code:script.js
async function fetchAllCards(project, verbose) {
const pageNum = await getPageCount(project); //取得するページ数
const maxIndex = pageNum > 1000 ? Math.floor(pageNum / 1000) + 1 : 1; // APIを叩く回数
const pages = []; // ページ情報を格納する配列
// 一気にAPIを叩いてページ情報を取得する
.map(async (_, index) => {
const res = await fetch(/api/pages/${project}/?limit=1000&skip=${index*1000});
const json = await res.json();
return json.pages;
});
const rawPages = (await Promise.all(promises)).flat();
return rawPages.map(({title, image, descriptions}) => {return {title, image, descriptions};});
}
code:script.js
async function getProjectUpdated(project, verbose) {
try {
const response = await fetch(/api/projects/${project});
if(!response.ok) throw Error(Invalid response. status code: ${response.status});
const {updated} = await response.json();
return updated;
} catch(e) {
throw e;
}
}
code:script.js
async function getPageCount(project, verbose) {
try {
const response = await fetch(/api/pages/${project}/?limit=1);
if(!response.ok) throw Error(Invalid response. status code: ${response.status});
const json = await response.json();
return parseInt(json.count);
} catch(e) {
throw e;
}
}
ログ出力
code:script.js
function _log(functionName, verbose, ...messages) {
if (!verbose) return;
console.log([${functionName}@scrapbox-cache-fetch], ...messages);
}
テストコード
code:js
(async () => {
const {execute} = await import('/api/code/takker/scrapbox-cache-fetch/test1.js');
await execute();
})();
code:test1.js
import {getAllLinks} from '/api/code/takker/scrapbox-cache-fetch/script.js';
export const projects = [
'hub',
'shokai',
'nishio',
'masui',
'motoso',
'villagepump',
'rashitamemo',
'rakusai',
'yuiseki',
'june29',
'ucdktr2016',
'thinkandcreateteck',
'customize',
'scrapboxlab',
'scrasobox',
'foldrr',
'scrapbox-drinkup',
'public-mrsekut',
'mrsekut-p',
'marshmallow-rm',
'wkpmm',
'sushitecture',
'nwtgck',
'dojineko',
'kadoyau',
'inteltank',
'sta',
'kn1cht',
'miyamonz',
'rmaruon',
'MISONLN41',
'yuta0801',
'choiyakiBox',
'choiyaki-hondana',
'spud-oimo',
'keroxp',
'aioilight',
];
export const execute = async ({maxAge = 60, verbose= true} = {}) => {
console.log(await getAllLinks(projects, {maxAge, verbose}));
};