scrapbox-link-database

2021-06-19 16:09:14 なんか動かなくなってしまった

代わりにもっとシンプルなinterfaceのやつを作った

/icons/hr.icon

複数のUserScriptから参照できるようにする

APIのcacheとして用いる

仕組み

cache戦略

data structure

cache-links

project: string

scrapbox project name

これをkeyにする

fetched: Date

データを取得した日時

index化しておく

pages: Page[]

link情報

code:ts

type Page = {

title: string; // ページタイトル

links: string[] // 内部リンクのリスト

}

cache-icons

画像つきページのみを格納したもの

project: string

scrapbox project name

これをkeyにする

updated: Date

projectの更新日時

index化しておく

pages: string[]

画像つきページのタイトル

2021-06-19

12:03:22 久々に実行したらエラーが出た

code:log

Uncaught (in promise) DOMException: An attempt was made to use an object that is not, or is no longer, usable

検索してもよくわからん

作り直すか。

2021-03-16 05:32:44 とりあえず完成

code:script.js

import { openDB } from '../idb/with-async-ittr.js';

import {

fetchPages,

fetchLinks,

getProjectUpdated

} from '/api/code/takker/scrapbox-api-helper/scrapboxAPI.js';

import {sub, getUnixTime} from '../date-fns.min.js/script.js';

export class CacheStorage {

constructor(expired = 3600) {

this._expired = expired;

// databaseを初期化する

this._initialized = (async () => {

this._db = await openDB(CacheStorage.name, CacheStorage.version, {

// database の更新処理

upgrade(db) {

// Object Storeをすべて消す

db.deleteObjectStore(storeName)

);

// object storeを作る

// link data

const linkStore = db.createObjectStore(CacheStorage.linkStoreName, {

keyPath: 'project',

});

linkStore.createIndex('fetched', 'fetched');

// icon data

const iconStore = db.createObjectStore(CacheStorage.iconStoreName, {

keyPath: 'project',

});

iconStore.createIndex('fetched', 'fetched');

},

});

})();

}

static name = 'UserScript';

static version = 4;

static linkStoreName = 'cache-links';

static iconStoreName = 'cache-icons';

async get(projects, {icon = false, reload = false} = {}) {

// 一旦更新処理をしてから取得する

await this.update(projects, {icon, reload});

// databaseから取ってくる

const result = [];

await this._transaction(

store => Promise.all(

projects.map(async project => {

const {titles, pages} = await store.get(project);

result.push(icon ? {project, titles} : {project, pages});

})

),

{icon, mode: 'readonly'},

);

return result;

}

async clear(projects, {icon = false} = {}) {

await this.waitForInitialization();

await this._transaction(

store => Promise.all(projects.map(project => tx.store.delete(project))),

{icon, mode: 'readwrite'}

);

}

// databaseの構築が終わるまで待機する

async waitForInitialization() {

await this._initialized;

}

async update(projects, {icon = false, reload = false} = {}) {

await this.waitForInitialization();

const now = new Date();

// 更新を取得するproject listを作る

// projectsに含まれるもののみ対象とする

const projectUpdateds = [];

await this._transaction(async store => {

// cacheのないprojectを先に入れておく

const availableProjects = await store.getAllKeys();

projectUpdateds.push(...projects.flatMap(project => {

}));

const index = store.index('fetched');

const iterator = reload ?

index.iterate() :

// fetched + this._expired < 現在時刻であるもののみ更新対象とする

index.iterate(

IDBKeyRange.upperBound(sub(now, {seconds: this._expired}), true)

);

05:24:06 何故か同じ値が二回繰り返される

05:23:59 直った！

code:script.js

for await (const cursor of iterator) {

const updateData = cursor.value;

if (!projects.includes(updateData.project)) continue;

projectUpdateds.push({

project: updateData.project,

prevFetched: getUnixTime(updateData.fetched), // 秒単位のtimestampに変換しておく

});

}

//await tx.done;

}, {icon, mode: 'readonly'});

// 先にfetchedを更新しておく

await this._transaction(async store => {

await Promise.all(projectUpdateds.flatMap(async ({project, prevFetched}) => {

if (prevFetched === 0) return [];

const {fetched, ...rest} = await store.get(project);

}));

}, {icon, mode: 'readwrite'});

// 更新する必要のあるproject listを作る

const targetProjects = (await Promise.all(

projectUpdateds.map(async ({project, prevFetched}) => {

const fetched = await getProjectUpdated(project);

})

)).flat();

// networkからdataを取得する

const data = await Promise.all(targetProjects.map(async project => icon ?

{project, titles: await fetchAllIcons(project)} :

{project, pages: await fetchAllLinks(project)}

));

// transactionを開いてdataを格納する

await this._transaction(async store => {

const keys = await store.getAllKeys();

await Promise.all(

data.map(({project, titles, pages}) =>

keys.includes(project) ?

store.put(icon ? {project, titles, fetched: now} : {project, pages, fetched: now}) :

store.add(icon ? {project, titles, fetched: now} : {project, pages, fetched: now})

)

)

}, {icon, mode: 'readwrite'});

// 更新したproject listを返す

return data.map(({project}) => project);

}

async _transaction(callback, {icon, mode}) {

const storeName = icon ? CacheStorage.iconStoreName : CacheStorage.linkStoreName;

const tx = this._db.transaction(storeName, mode);

const store = tx.objectStore(storeName);

await callback(store);

await tx.done;

}

}

code:script.js

async function fetchAllIcons(project) {

const pages = await fetchPages({project});

}

async function fetchAllLinks(project) {

const pages = await fetchLinks({project});

return pages.map(({title, links}) => {return {title, links};});

}

test code

1. methodの動作チェック

2021-03-16 04:01:26 問題なさそう

code:js

import('/api/code/programming-notes/scrapbox-link-database/test1.js');

code:test1.js

import {CacheStorage} from './script.js';

const storage = new CacheStorage(300); // cacheの寿命を5minにする

window.cacheStorage = storage;

2. 大量にデータを格納するテスト

04:30:06 問題なさそう

https://gyazo.com/0c636746488c85c003f22af87c6f9c20

code:js

import('/api/code/programming-notes/scrapbox-link-database/test2.js');

code:test2.js

import {CacheStorage} from './script.js';

import {projects, icons} from './list.js';

const storage = new CacheStorage(300); // cacheの寿命を5minにする

(async () => {

console.log(await storage.get(projects));

})();

(async () => {

console.log(await storage.get(icons, {icon: true}));

})();

window.cacheStorage = storage;

code:list.js

export const projects = [

'hub',

'shokai',

'nishio',

'masui',

'rakusai',

'yuiseki',

'june29',

'villagepump',

'rashitamemo',

'thinkandcreateteck',

'customize',

'scrapboxlab',

'scrasobox',

'foldrr',

'scrapbox-drinkup',

'remote',

'motoso',

'public-mrsekut',

'mrsekut-p',

'marshmallow-rm',

'wkpmm',

'sushitecture',

'nwtgck',

'dojineko',

'kadoyau',

'inteltank',

'sta',

'kn1cht',

'miyamonz',

'rmaruon',

'yuta0801',

'aviutl',

'ePi5131',

'choiyakiBox',

'choiyaki-hondana',

'suto3',

'spud-oimo',

'imo-memo',

'keroxp',

'aioilight',

'NDLSH-SB',

'kuuote',

'programming-notes',

'JavaScriptTips',

'yanma',

'yutaro',

'ci7lus',

'nota-techconf',

];

確認していないこと

WebWorkerからのアクセス

複数タブからのアクセス

JavaScript.icon