scrapbox-link-database@0.2.0
複数のUserScriptから参照できるようにする
APIのcacheとして用いる
使い方
code:js
import {get, clear} from '/api/code/programming-notes/scrapbox-link-database@0.2.0/script.js';
// projectのリンク情報を取得する
// projectのリンク情報を削除する
classは止める
bundleしづらくなる
コードが複雑になる
仕組み
cache戦略
data structure
cache-links
project: string
scrapbox project name
これをkeyにする
fetched: Date
データを取得した日時
index化しておく
pages: Page[]
link情報
code:ts
type Page = {
title: string; // ページタイトル
hasIcon: boolean; // アイコンが有るかどうか
links: string[] // 内部リンクのリスト
}
一度に情報を取得できる
実装したいこと
/icons/done.icon補完ソースの生成処理をworkerに移譲する
リンクデータのfetchにかなり時間がかかるので、UI threadをblockしないようにしたい
update()をworkerに移す
2021-06-27
12:05:57 リンクデータのfetchをweb workerに移譲した
text/javascriptもapplication/javascriptもだめ
JS fileから読み込んだときは問題ない
Promiseを返そうとしてもこのエラーが出るっぽいな
11:26:58 設定変数を別ファイルに切り出した
web workerからも使いたかったので
2021-06-19
22:10:33 getのintefaceを変える
更新されたかどうかを表すフラグを返り値に追加する
15:59:12 存在しないprojectのidは除外する
↓の原因はこれだった
14:40:12 どうやらdataがあるものとないものとがあるらしい
ないやつは、データ取得に失敗している?
どのprojectだろう?
データのないやつを一覧してみる
15:29:14 参加しているprojectsの情報が取得されていなかった
こうしたら取得されるようになった
演算子の優先順位が変だったみたい
code:diff
- ({name, id, updated}) =>
- updated > projectUpdateds.find(data => data.id === id)?.prevFetched ?? 0 ?
- []
- );
+ ({name, id, updated}) => {
+ const prevFetched = (projectUpdateds.find(data => data.id === id)?.prevFetched ?? 0);
+ console.log({name, updated, prevFetched});
+ });
でも何故かdatabaseには保存されない……
databaseを見てみると、cacheされているprojectとされていないprojectとがある
get()で返されるprojectのなかで、pagesがundefinedになるのはやはりcacheされていないprojectだけだ
22 projectsが該当する
ここから参加しているprojectを除いて取得できるようにコードを変えよう
15:47:09 変えた
それでもどの参加しているprojectもcacheから読み取れない……
まさか容量超過?
でもこの程度のデータ容量で超過するわけないのだが……
14:31:24 await store.get(id)でデータは取得できている
dev toolでも確認できる
なのにそこからdataを更に取り出そうとするとundefinedに変わってしまう……
14:03:46 APIを叩きすぎてtimeoutしてしまったので、先にそっちを調節する
13:51:24 keyをprojectからprojectIdに変更した
……のだが、何故かdatabaseからのデータ取得に失敗する
原因を調べる
13:27:02 とりあえず完成
dataのfetchはもう少し工夫したほうがいいかも
serverに負担がかからないように少しずらす
dependencies
code:script.js
import { openDB } from '../idb/with-async-ittr.js';
import {src} from './workerSrc.js';
import {DBName, Version, StoreName} from './settings.js';
const worker = new Worker(src);
// databaseを初期化する
function initialize() {
return openDB(DBName, Version, {
// 更新処理
upgrade(db) {
// Object Storeをすべて消す
db.deleteObjectStore(storeName)
);
// object storeを作り直す
const store = db.createObjectStore(StoreName, {
keyPath: 'id',
});
store.createIndex('fetched', 'fetched');
},
});
}
// 初期化
const getDB = initialize();
export async function get(projectIds, options) {
// 予め重複を消しておく
if (projectIds.length === 0) return {};
// 先に更新する
const hasUpdate = await update(projectIds, options);
// databaseから取ってくる
const db = await getDB;
const tx = db.transaction(StoreName, 'readwrite');
const store = tx.objectStore(StoreName);
const result = await Promise.all(projectIds.map(async id => {
const data = await store.get(id);
return data ? {id, project: data.project, pages: data.pages} : undefined;
}));
await tx.done;
return {data: result.filter(data => data), hasUpdate};
}
// cacheを消す
export async function clear(projectIds) {
// cacheを削除する
const db = await getDB;
const tx = db.transaction(StoreName, 'readwrite');
const store = tx.objectStore(StoreName);
await Promise.all(projectIds.map(id => store.delete(id)));
await tx.done;
}
// dataを更新する
// reloadをつけるとprojects全てをnetworkから取得し直すが、更新が見つからなければ何もしない
function update(projectIds, options) {
return new Promise(resolve => {
const handleMessage = ({data}) => {
worker.removeEventListener('message', handleMessage);
resolve(data);
};
worker.addEventListener('message', handleMessage);
worker.postMessage({projectIds, options});
});
}
code:settings.js
export const StoreName = 'cache-links';
export const DBName = 'UserScript';
export const Version = 7;
code:workerSrc.js
export const src = '/api/code/programming-notes/scrapbox-link-database@0.2.0/worker_min.js';
code:worker_min.js
(()=>{var L=(t,r)=>r.some(n=>t instanceof n),me,le;function Fe(){return me||(me=IDBDatabase,IDBObjectStore,IDBIndex,IDBCursor,IDBTransaction)}function Le(){return le||(le=IDBCursor.prototype.advance,IDBCursor.prototype.continue,IDBCursor.prototype.continuePrimaryKey)}var ce=new WeakMap,V=new WeakMap,pe=new WeakMap,G=new WeakMap,W=new WeakMap;function Pe(t){let r=new Promise((n,i)=>{let d=()=>{t.removeEventListener("success",s),t.removeEventListener("error",u)},s=()=>{n(v(t.result)),d()},u=()=>{i(t.error),d()};t.addEventListener("success",s),t.addEventListener("error",u)});return r.then(n=>{n instanceof IDBCursor&&ce.set(n,t)}).catch(()=>{}),W.set(r,t),r}function He(t){if(V.has(t))return;let r=new Promise((n,i)=>{let d=()=>{t.removeEventListener("complete",s),t.removeEventListener("error",u),t.removeEventListener("abort",u)},s=()=>{n(),d()},u=()=>{i(t.error||new DOMException("AbortError","AbortError")),d()};t.addEventListener("complete",s),t.addEventListener("error",u),t.addEventListener("abort",u)});V.set(t,r)}var Z={get(t,r,n){if(t instanceof IDBTransaction){if(r==="done")return V.get(t);if(r==="objectStoreNames")return t.objectStoreNames||pe.get(t);if(r==="store")return n.objectStoreNames1?void 0:n.objectStore(n.objectStoreNames0)}return v(tr)},set(t,r,n){return tr=n,!0},has(t,r){return t instanceof IDBTransaction&&(r==="done"||r==="store")?!0:r in t}};function P(t){Z=t(Z)}function Re(t){return t===IDBDatabase.prototype.transaction&&!("objectStoreNames"in IDBTransaction.prototype)?function(r,...n){let i=t.call(k(this),r,...n);return pe.set(i,r.sort?r.sort():r),v(i)}:Le().includes(t)?function(...r){return t.apply(k(this),r),v(ce.get(this))}:function(...r){return v(t.apply(k(this),r))}}function Qe(t){return typeof t=="function"?Re(t):(t instanceof IDBTransaction&&He(t),L(t,Fe())?new Proxy(t,Z):t)}function v(t){if(t instanceof IDBRequest)return Pe(t);if(G.has(t))return G.get(t);let r=Qe(t);return r!==t&&(G.set(t,r),W.set(r,t)),r}var k=t=>W.get(t);function ge(t,r,{blocked:n,upgrade:i,blocking:d,terminated:s}={}){let u=indexedDB.open(t,r),m=v(u);return i&&u.addEventListener("upgradeneeded",f=>{i(v(u.result),f.oldVersion,f.newVersion,v(u.transaction))}),n&&u.addEventListener("blocked",()=>n()),m.then(f=>{s&&f.addEventListener("close",()=>s()),d&&f.addEventListener("versionchange",()=>d())}).catch(()=>{}),m}var $e="get","getKey","getAll","getAllKeys","count",Be="put","add","delete","clear",K=new Map;function xe(t,r){if(!(t instanceof IDBDatabase&&!(r in t)&&typeof r=="string"))return;if(K.get(r))return K.get(r);let n=r.replace(/FromIndex$/,""),i=r!==n,d=Be.includes(n);if(!(n in(i?IDBIndex:IDBObjectStore).prototype)||!(d||$e.includes(n)))return;let s=async function(u,...m){let f=this.transaction(u,d?"readwrite":"readonly"),p=f.store;return i&&(p=p.index(m.shift())),(await Promise.all([pn(...m),d&&f.done]))0};return K.set(r,s),s}P(t=>({...t,get:(r,n,i)=>xe(r,n)||t.get(r,n,i),has:(r,n)=>!!xe(r,n)||t.has(r,n)}));var ze="continue","continuePrimaryKey","advance",he={},J=new WeakMap,ve=new WeakMap,Xe={get(t,r){if(!ze.includes(r))return tr;let n=her;return n||(n=her=function(...i){J.set(this,ve.get(this)r(...i))}),n}};async function*Ve(...t){let r=this;if(r instanceof IDBCursor||(r=await r.openCursor(...t)),!r)return;r=r;let n=new Proxy(r,Xe);for(ve.set(n,r),W.set(n,k(r));r;)yield n,r=await(J.get(n)||r.continue()),J.delete(n)}function De(t,r){return r===Symbol.asyncIterator&&L(t,IDBIndex,IDBObjectStore,IDBCursor)||r==="iterate"&&L(t,IDBIndex,IDBObjectStore)}P(t=>({...t,get(r,n,i){return De(r,n)?Ve:t.get(r,n,i)},has(r,n){return De(r,n)||t.has(r,n)}}));async function je({project:t}){let r=null,n=[],i=[];do{let d=await fetch(r?/api/pages/${t}/search/titles?followingId=${r}:/api/pages/${t}/search/titles);r=d.headers.get("X-Following-Id"),i.push(d.json().then(s=>n.push(...s)))}while(r);return await Promise.all(i),n}function o(t){if(t===null||t===!0||t===!1)return NaN;var r=Number(t);return isNaN(r)?r:r<0?Math.ceil(r):Math.floor(r)}function e(t,r){if(r.length<t)throw new TypeError(t+" argument"+(t>1?"s":"")+" required, but only "+r.length+" present")}function a(t){e(1,arguments);let r=Object.prototype.toString.call(t);return t instanceof Date||typeof t=="object"&&r==="object Date"?new Date(t.getTime()):typeof t=="number"||r==="object Number"?new Date(t):((typeof t=="string"||r==="object String")&&typeof console!="undefined"&&(console.warn("Starting with v2.0.0-beta.1 date-fns doesn't accept strings as arguments. Please use parseISO to parse strings. See: https://git.io/fjule"),console.warn(new Error().stack)),new Date(NaN))}function D(t,r){e(2,arguments);var n=a(t),i=o(r);return isNaN(i)?new Date(NaN):(i&&n.setDate(n.getDate()+i),n)}function O(t,r){e(2,arguments);var n=a(t),i=o(r);if(isNaN(i))return new Date(NaN);if(!i)return n;var d=n.getDate(),s=new Date(n.getTime());s.setMonth(n.getMonth()+i+1,0);var u=s.getDate();return d>=u?s:(n.setFullYear(s.getFullYear(),s.getMonth(),d),n)}function C(t){return function(r){var n=r||{},i=n.width?String(n.width):t.defaultWidth,d=t.formatsi||t.formatst.defaultWidth;return d}}var st={full:"EEEE, MMMM do, y",long:"MMMM do, y",medium:"MMM d, y",short:"MM/dd/yyyy"},dt={full:"h:mm:ss a zzzz",long:"h:mm:ss a z",medium:"h:mm:ss a",short:"h:mm a"},ut={full:"{{date}} 'at' {{time}}",long:"{{date}} 'at' {{time}}",medium:"{{date}}, {{time}}",short:"{{date}}, {{time}}"},Qs={date:C({formats:st,defaultWidth:"full"}),time:C({formats:dt,defaultWidth:"full"}),dateTime:C({formats:ut,defaultWidth:"full"})};function I(t){return function(r,n){var i=n||{},d=i.context?String(i.context):"standalone",s;if(d==="formatting"&&t.formattingValues){let m=t.defaultFormattingWidth||t.defaultWidth,f=i.width?String(i.width):m;s=t.formattingValuesf||t.formattingValuesm}else{let m=t.defaultWidth,f=i.width?String(i.width):t.defaultWidth;s=t.valuesf||t.valuesm}var u=t.argumentCallback?t.argumentCallback(r):r;return su}}var ft={narrow:"B","A",abbreviated:"BC","AD",wide:"Before Christ","Anno Domini"},mt={narrow:"1","2","3","4",abbreviated:"Q1","Q2","Q3","Q4",wide:"1st quarter","2nd quarter","3rd quarter","4th quarter"},lt={narrow:"J","F","M","A","M","J","J","A","S","O","N","D",abbreviated:"Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec",wide:"January","February","March","April","May","June","July","August","September","October","November","December"},ct={narrow:"S","M","T","W","T","F","S",short:"Su","Mo","Tu","We","Th","Fr","Sa",abbreviated:"Sun","Mon","Tue","Wed","Thu","Fri","Sat",wide:"Sunday","Monday","Tuesday","Wednesday","Thursday","Friday","Saturday"},pt={narrow:{am:"a",pm:"p",midnight:"mi",noon:"n",morning:"morning",afternoon:"afternoon",evening:"evening",night:"night"},abbreviated:{am:"AM",pm:"PM",midnight:"midnight",noon:"noon",morning:"morning",afternoon:"afternoon",evening:"evening",night:"night"},wide:{am:"a.m.",pm:"p.m.",midnight:"midnight",noon:"noon",morning:"morning",afternoon:"afternoon",evening:"evening",night:"night"}},gt={narrow:{am:"a",pm:"p",midnight:"mi",noon:"n",morning:"in the morning",afternoon:"in the afternoon",evening:"in the evening",night:"at night"},abbreviated:{am:"AM",pm:"PM",midnight:"midnight",noon:"noon",morning:"in the morning",afternoon:"in the afternoon",evening:"in the evening",night:"at night"},wide:{am:"a.m.",pm:"p.m.",midnight:"midnight",noon:"noon",morning:"in the morning",afternoon:"in the afternoon",evening:"in the evening",night:"at night"}};function xt(t,r){var n=Number(t),i=n%100;if(i>20||i<10)switch(i%10){case 1:return n+"st";case 2:return n+"nd";case 3:return n+"rd"}return n+"th"}var Vs={ordinalNumber:xt,era:I({values:ft,defaultWidth:"wide"}),quarter:I({values:mt,defaultWidth:"wide",argumentCallback:function(t){return Number(t)-1}}),month:I({values:lt,defaultWidth:"wide"}),day:I({values:ct,defaultWidth:"wide"}),dayPeriod:I({values:pt,defaultWidth:"wide",formattingValues:gt,defaultFormattingWidth:"wide"})};function ie(t){return function(r,n){var i=String(r),d=n||{},s=i.match(t.matchPattern);if(!s)return null;var u=s0,m=i.match(t.parsePattern);if(!m)return null;var f=t.valueCallback?t.valueCallback(m0):m0;return f=d.valueCallback?d.valueCallback(f):f,{value:f,rest:i.slice(u.length)}}}function S(t){return function(r,n){var i=String(r),d=n||{},s=d.width,u=s&&t.matchPatternss||t.matchPatternst.defaultMatchWidth,m=i.match(u);if(!m)return null;var f=m0,p=s&&t.parsePatternss||t.parsePatternst.defaultParseWidth,g;return Object.prototype.toString.call(p)==="object Array"?g=vt(p,function(l){return l.test(f)}):g=ht(p,function(l){return l.test(f)}),g=t.valueCallback?t.valueCallback(g):g,g=d.valueCallback?d.valueCallback(g):g,{value:g,rest:i.slice(f.length)}}}function ht(t,r){for(var n in t)if(t.hasOwnProperty(n)&&r(tn))return n}function vt(t,r){for(var n=0;n<t.length;n++)if(r(tn))return n}var Dt=/^(\d+)(th|st|nd|rd)?/i,jt=/\d+/i,wt={narrow:/^(b|a)/i,abbreviated:/^(b\.?\s?c\.?|b\.?\s?c\.?\s?e\.?|a\.?\s?d\.?|c\.?\s?e\.?)/i,wide:/^(before christ|before common era|anno domini|common era)/i},bt={any:/^b/i,/^(a|c)/i},yt={narrow:/^1234/i,abbreviated:/^q1234/i,wide:/^1234(th|st|nd|rd)? quarter/i},Ot={any:/1/i,/2/i,/3/i,/4/i},Tt={narrow:/^jfmasond/i,abbreviated:/^(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)/i,wide:/^(january|february|march|april|may|june|july|august|september|october|november|december)/i},It={narrow:/^j/i,/^f/i,/^m/i,/^a/i,/^m/i,/^j/i,/^j/i,/^a/i,/^s/i,/^o/i,/^n/i,/^d/i,any:/^ja/i,/^f/i,/^mar/i,/^ap/i,/^may/i,/^jun/i,/^jul/i,/^au/i,/^s/i,/^o/i,/^n/i,/^d/i},St={narrow:/^smtwf/i,short:/^(su|mo|tu|we|th|fr|sa)/i,abbreviated:/^(sun|mon|tue|wed|thu|fri|sat)/i,wide:/^(sunday|monday|tuesday|wednesday|thursday|friday|saturday)/i},Mt={narrow:/^s/i,/^m/i,/^t/i,/^w/i,/^t/i,/^f/i,/^s/i,any:/^su/i,/^m/i,/^tu/i,/^w/i,/^th/i,/^f/i,/^sa/i},kt={narrow:/^(a|p|mi|n|(in the|at) (morning|afternoon|evening|night))/i,any:/^(ap\.?\s?m\.?|midnight|noon|(in the|at) (morning|afternoon|evening|night))/i},_t={any:{am:/^a/i,pm:/^p/i,midnight:/^mi/i,noon:/^no/i,morning:/morning/i,afternoon:/afternoon/i,evening:/evening/i,night:/night/i}},td={ordinalNumber:ie({matchPattern:Dt,parsePattern:jt,valueCallback:function(t){return parseInt(t,10)}}),era:S({matchPatterns:wt,defaultMatchWidth:"wide",parsePatterns:bt,defaultParseWidth:"any"}),quarter:S({matchPatterns:yt,defaultMatchWidth:"wide",parsePatterns:Ot,defaultParseWidth:"any",valueCallback:function(t){return t+1}}),month:S({matchPatterns:Tt,defaultMatchWidth:"wide",parsePatterns:It,defaultParseWidth:"any"}),day:S({matchPatterns:St,defaultMatchWidth:"wide",parsePatterns:Mt,defaultParseWidth:"any"}),dayPeriod:S({matchPatterns:kt,defaultMatchWidth:"any",parsePatterns:_t,defaultParseWidth:"any"})};var Am=24*60*60*1e3;function B(t){e(1,arguments);var r=a(t),n=r.getTime();return n}function z(t){return e(1,arguments),Math.floor(B(t)/1e3)}function U(t,r){e(2,arguments);var n=o(r);return D(t,-n)}function X(t,r){e(2,arguments);var n=o(r);return O(t,-n)}function F(t,r){if(e(2,arguments),!r||typeof r!="object")return new Date(NaN);let n="years"in r?o(r.years):0,i="months"in r?o(r.months):0,d="weeks"in r?o(r.weeks):0,s="days"in r?o(r.days):0,u="hours"in r?o(r.hours):0,m="minutes"in r?o(r.minutes):0,f="seconds"in r?o(r.seconds):0,p=X(a(t),i+n*12),g=U(p,s+d*7),l=m+u*60,h=(f+l*60)*1e3;return new Date(g.getTime()-h)}var Dr=Math.pow(10,8)*24*60*60*1e3,qv=-Dr;async function Ye(t,{includeJoined:r=!0}={}){let n=Math.floor(t.length/100)+1,d=(await Promise.all(...Array(n).keys().map(async s=>{let u=new URLSearchParams;t.slice(s*100,100+s*100).forEach(p=>u.append("ids",p));let m=await fetch(/api/projects?${u.toString()}),{projects:f}=await m.json();return f}))).flat();return r?...new Set(d.map(({id:u})=>u)).map(u=>d.find(m=>m.id===u)):...new Set(d.map(({id:s})=>s)).filter(s=>t.some(u=>u===s)).map(s=>d.find(u=>u.id===s))}async function Ce(t,{maxInProgress:r=void 0}={}){if(!r||r<0||t.length<=r)return await Promise.all(t.map(d=>d()));let n=t.map(d=>!1),i=[];return await Promise.all(...Array(r).keys().map(async d=>{do nd=!0,id=await td(),d=n.findIndex(s=>!s);while(d!==-1)})),i}var M="cache-links",Ne="UserScript",Ee=7;var jr=ge(Ne,Ee);self.addEventListener("message",async({data:{projectIds:t,options:r}})=>{self.postMessage(await wr(await jr,t,r))});async function wr(t,r,n){let{reload:i=!1,expired:d=3600}=n??{},s=[],u=new Date;{let l=t.transaction(M,"readonly"),c=l.objectStore(M),h=await c.getAllKeys();s.push(...r.flatMap(j=>h.includes(j)?[]:{id:j,prevFetched:0}));let x=c.index("fetched"),y=i?x.iterate():x.iterate(IDBKeyRange.upperBound(F(u,{seconds:d}),!0));for await(let j of y){let{id:fe,fetched:Ue}=j.value;!r.includes(fe)||s.push({id:fe,prevFetched:z(Ue)})}await l.done}if(s.length===0)return!1;{let l=t.transaction(M,"readwrite"),c=l.objectStore(M);await Promise.all(s.flatMap(async({id:h,prevFetched:x})=>{if(x===0)return[];let{fetched:y,...j}=await c.get(h);returnawait c.put({fetched:u,...j})})),await l.done}let f=(await Ye(s.map(({id:l})=>l),{includeJoined:!1})).flatMap(({name:l,id:c,updated:h})=>{let x=s.find(y=>y.id===c)?.prevFetched??0;return h>x?{project:l,id:c}:[]});if(f.length===0)return!1;let p=0,g=await Ce(f.map(({project:l,id:c})=>async()=>{let h=p++;console.log([${h}/${f.length}] Fetching pages from /${l}...);let x=await je({project:l});return console.log([${h}/${f.length}] Fetched pages from /${l}.),{project:l,id:c,pages:x}}),{maxInProgress:10});{let l=t.transaction(M,"readwrite"),c=l.objectStore(M),h=await c.getAllKeys();await Promise.all(g.map(({id:x,project:y,pages:j})=>h.includes(x)?c.put({id:x,project:y,pages:j,fetched:u}):c.add({id:x,project:y,pages:j,fetched:u}))),await l.done}return!0}})(); code:sh
code:build.ts
import {run} from '/api/code/takker/UserScriptをbundleするDeno_script/script.ts';
import {BlobToURI} from '/api/code/takker/BlobをData_URIに変換する/script.js';
const {outputFiles} = await run(
https://scrapbox.io/api/code/programming-notes/scrapbox-link-database@0.2.0/worker.js,
{
},
{
charset: 'utf8',
bundle: true,
minify: true,
write: false, // 標準出力やfileにbundleしたコードを出力しない
format: 'iife',
}
);
console.log(outputFiles?.0?.text ?? ''); code:worker.js
import { openDB } from '../idb/with-async-ittr.js';
import {
fetchLinks,
getProjectUpdated,
} from '/api/code/takker/scrapbox-api-helper/scrapboxAPI.js';
import {sub, getUnixTime} from '../date-fns.min.js/script.js';
import {getProjectInfo} from '../scrapboxのproject情報を一括して取得するUserScript/script.js';
import {parallel} from '../promise-parallel-throttle/script.js';
import {DBName, Version, StoreName} from './settings.js';
const getDB = openDB(DBName, Version);
self.addEventListener('message', async ({data: {projectIds, options}}) => {
self.postMessage(await update(await getDB, projectIds, options));
});
// dataを更新する
// reloadをつけるとprojects全てをnetworkから取得し直すが、更新が見つからなければ何もしない
async function update(db, projectIds, options) {
const {reload = false, expired = 3600,} = options ?? {};
// 更新対象のproject listを作る
// projectsに含まれるもののみを更新対象とする
const projectUpdateds = [];
const now = new Date();
{
const tx = db.transaction(StoreName, 'readonly');
const store = tx.objectStore(StoreName);
const cachedProjects = await store.getAllKeys();
// cacheのないprojectを先に入れておく
projectUpdateds.push(...projectIds.flatMap(id => {
}));
const index = store.index('fetched');
const iterator = reload ?
// 再読み込みする場合はすべてのprojectを更新対象にする
index.iterate() :
// fetched + expired < 現在時刻であるもののみ更新対象とする
index.iterate(
IDBKeyRange.upperBound(sub(now, {seconds: expired}), true)
);
for await (const cursor of iterator) {
const {id, fetched} = cursor.value;
if (!projectIds.includes(id)) continue;
projectUpdateds.push({
id,
prevFetched: getUnixTime(fetched), // 秒単位のtimestampに変換しておく
});
}
await tx.done;
}
if (projectUpdateds.length === 0) return false;
// 先に更新日時を書き込んでおく
{
const tx = db.transaction(StoreName, 'readwrite');
const store = tx.objectStore(StoreName);
await Promise.all(projectUpdateds.flatMap(async ({id, prevFetched}) => {
if (prevFetched === 0) return [];
const {fetched, ...rest} = await store.get(id);
}));
await tx.done;
}
// 実際に更新する必要のあるproject listを作る
const projectInfos = await getProjectInfo(projectUpdateds.map(({id}) => id), {includeJoined: false});
const targetProjects = projectInfos
.flatMap(
({name, id, updated}) => {
const prevFetched = (projectUpdateds.find(data => data.id === id)?.prevFetched ?? 0);
});
if (targetProjects.length === 0) return false;
// networkからdataを取得する
let counter = 0;
const data = await parallel(targetProjects.map(({project, id}) => async () => {
const index = counter++;
console.log([${index}/${targetProjects.length}] Fetching pages from /${project}...);
const pages = await fetchLinks({project});
console.log([${index}/${targetProjects.length}] Fetched pages from /${project}.);
return {project, id, pages};
}
), {maxInProgress: 10});
// dataを格納する
{
const tx = db.transaction(StoreName, 'readwrite');
const store = tx.objectStore(StoreName);
const keys = await store.getAllKeys();
await Promise.all(
data.map(({id, project, pages}) =>
keys.includes(id) ?
store.put({id, project, pages, fetched: now}) :
store.add({id, project, pages, fetched: now})
)
);
await tx.done;
}
return true;
}
test code
code:js
import('/api/code/programming-notes/scrapbox-link-database@0.2.0/test.js');
code:test.js
import {get, clear} from './script.js';
const watchListIds = Object.keys(JSON.parse(localStorage.getItem('projectsLastAccessed')));
window.getLinks = async (options) => await get(watchListIds, options);
window.clearLinks = async (options) => await clear(watchListIds);