/** * A shared folder and its repository, kept in step. The notes live here, * so the page is as quick as a personal vault; the repository is where * they are kept, so they are plain markdown files anyone can clone, push * to, or open in another editor. * * One pass (`syncSpace`) reads, then writes: * * read when the branch has moved since the last read, walk the folder * and take every file that changed. A note nobody has touched * here takes the repository's words; a note edited here *and* * there keeps ours, and the repository's version is kept beside * it as " (repo copy)" — nothing is ever lost. * write whatever was written, renamed, moved or removed here since, * as one commit. * * A note knows what the repository last had (synced_path and the git blob * of its words, synced_oid), so "changed here" is a comparison, never a * flag to forget. Commits go through the forge's API with a token * (NOTES_REPO_TOKEN): the app's own token reads only. * * Writes are owed, not immediate: a change waits a few seconds for the * next (typing is many saves), and a slow trickle still commits within * MAX_WAIT_MS. Every space is also read once a minute. */ import type { Database } from "bun:sqlite"; import { createHash } from "node:crypto"; import type { Note } from "./actions.ts"; import { BODY_MAX } from "./actions.ts"; import { findSpace, spaceVault, tellSpace, titleOfPath, whenEdited, type Space } from "./spaces.ts"; /** The git blob id of some words: what `git hash-object` says of the file. */ export function blobOid(text: string): string { const bytes = Buffer.from(text, "utf8"); return createHash("sha1").update(`blob ${bytes.length}\0`).update(bytes).digest("hex"); } export interface Entry { readonly name: string; readonly type: "blob" | "tree" | string; readonly oid: string; readonly size: number; } /** The forge, as much of it as a shared folder needs. */ export interface Forge { /** The commit at the tip of a branch; null when there is none yet. */ head(repo: string, branch: string): Promise<string | null>; /** A folder's entries at a commit; null when it doesn't exist. */ list(repo: string, ref: string, path: string): Promise<Entry[] | null>; /** A file's words at a commit. */ read(repo: string, ref: string, path: string): Promise<string>; /** Write and remove files (`content: null`) as one commit on `branch`. */ commit( repo: string, branch: string, message: string, files: { path: string; content: string | null }[], ): Promise<string>; } type Fetch = (input: string, init?: RequestInit) => Promise<Response>; /** The forge's JSON API (`<forge>/api/v1`), as a token that may write the repositories. */ export function httpForge(api: string, token: string, fetch: Fetch = globalThis.fetch): Forge { const base = api.replace(/\/+$/, ""); const headers = { authorization: `Bearer ${token}`, "content-type": "application/json" }; const at = (repo: string, rest: string) => `${base}/repos/${repo}/${rest}`; const path = (p: string) => p.split("/").map(encodeURIComponent).join("/"); const failed = async (r: Response, what: string): Promise<never> => { throw new Error(`${what}: HTTP ${r.status} ${(await r.text()).slice(0, 200)}`); }; return { async head(repo, branch) { const r = await fetch(at(repo, `commits?ref=${encodeURIComponent(branch)}&limit=1`), { headers }); if (r.status === 404) throw new Error(`No repository ${repo} — or the token can't see it.`); if (!r.ok) return failed(r, "read the branch"); const log = (await r.json()) as { oid?: string }[]; return log[0]?.oid ?? null; }, async list(repo, ref, p) { const r = await fetch(at(repo, `contents${p === "" ? "" : `/${path(p)}`}?ref=${encodeURIComponent(ref)}`), { headers, }); if (r.status === 404) return null; if (!r.ok) return failed(r, `list ${p || "/"}`); const answer = (await r.json()) as { type?: string; entries?: Entry[] }; return answer.type === "dir" ? (answer.entries ?? []) : null; }, async read(repo, ref, p) { const r = await fetch(at(repo, `contents/${path(p)}?ref=${encodeURIComponent(ref)}`), { headers }); if (!r.ok) return failed(r, `read ${p}`); const answer = (await r.json()) as { encoding?: string; content?: string }; const content = answer.content ?? ""; return answer.encoding === "base64" ? Buffer.from(content, "base64").toString("utf8") : content; }, async commit(repo, branch, message, files) { const r = await fetch(at(repo, "edits"), { method: "POST", headers, body: JSON.stringify({ branch, message, files }), }); if (!r.ok) return failed(r, "commit"); const answer = (await r.json()) as { commit?: { commitOid?: string; oid?: string } | string }; const c = answer.commit; return typeof c === "string" ? c : (c?.commitOid ?? c?.oid ?? ""); }, }; } /** Folders a vault never holds: git's, and editors' own settings (`.obsidian`, `.vscode`). */ const hidden = (name: string): boolean => name.startsWith(".") || name === "node_modules"; /** Every markdown file under `folder`, path → blob id. */ async function walk(forge: Forge, space: Space, ref: string): Promise<Map<string, Entry>> { const files = new Map<string, Entry>(); const visit = async (dir: string, depth: number): Promise<void> => { if (depth > 12) return; const entries = await forge.list(space.repo, ref, dir); for (const e of entries ?? []) { if (hidden(e.name)) continue; const p = dir === "" ? e.name : `${dir}/${e.name}`; if (e.type === "tree") await visit(p, depth + 1); else if (e.type === "blob" && /\.md$/i.test(e.name) && e.size <= BODY_MAX) files.set(p, e); } }; await visit(space.folder, 0); return files; } const NOW = "strftime('%Y-%m-%dT%H:%M:%fZ', 'now')"; const notesOf = (db: Database, space: Space): Note[] => db.query<Note, [string]>("SELECT * FROM note WHERE author_id = ?1").all(spaceVault(space.id)); /** Whether the note has words, a name or a place the repository hasn't been given yet. */ export const unsynced = (n: Note): boolean => n.path !== null && (n.synced_path === null || n.synced_path !== n.path || n.synced_oid !== blobOid(n.body)); /** A title nobody in the vault has yet: `base`, else `base 2`, `base 3`, … */ function freeTitle(db: Database, vault: string, base: string): string { const taken = (t: string) => db .query<{ id: number }, [string, string]>("SELECT id FROM note WHERE author_id = ?1 AND title = ?2 COLLATE NOCASE") .get(vault, t) !== null; if (!taken(base)) return base; for (let n = 2; ; n++) if (!taken(`${base} ${n}`)) return `${base} ${n}`; } export interface Report { /** Notes taken from the repository: new, changed, removed. */ readonly pulled: number; /** Files the commit wrote or removed. */ readonly pushed: number; /** Repository versions kept beside a note edited in both places. */ readonly copies: number; readonly commit?: string; } /** Read what the repository has that we don't — when the branch has moved. */ async function pull(db: Database, forge: Forge, space: Space, head: string | null): Promise<{ pulled: number; copies: number }> { if (head === null || head === space.head) return { pulled: 0, copies: 0 }; const vault = spaceVault(space.id); const files = await walk(forge, space, head); const notes = notesOf(db, space); const bySynced = new Map(notes.filter((n) => n.synced_path !== null).map((n) => [n.synced_path!, n])); const byPath = new Map(notes.filter((n) => n.path !== null).map((n) => [n.path!.toLowerCase(), n])); const gone = new Set( db .query<{ path: string }, [number]>("SELECT path FROM space_gone WHERE space_id = ?1") .all(space.id) .map((g) => g.path), ); // Read every changed file first, then write them all at once. const wanted = [...files].filter(([p, e]) => bySynced.get(p)?.synced_oid !== e.oid && !gone.has(p)); const words = new Map<string, string>(); for (const [p] of wanted) words.set(p, await forge.read(space.repo, head, p)); let pulled = 0; let copies = 0; const changed: Note[] = []; const removed: number[] = []; db.transaction(() => { const keepCopy = (note: Note, body: string) => { const title = freeTitle(db, vault, `${note.title} (repo copy)`); const dir = note.path!.includes("/") ? note.path!.slice(0, note.path!.lastIndexOf("/") + 1) : ""; changed.push( db .query<Note, [string, string, string, string]>( "INSERT INTO note (title, body, author_id, author_name, path) VALUES (?1, ?2, ?3, 'the repository', ?4) RETURNING *", ) .get(title, body, vault, `${dir}${title}.md`)!, ); copies++; }; for (const [p, entry] of files) { if (gone.has(p)) continue; const known = bySynced.get(p); if (known !== undefined && known.synced_oid === entry.oid) continue; const body = words.get(p)!; // Made here and there under one name before either had seen the other. const twin = known ?? (byPath.get(p.toLowerCase())?.synced_path === null ? byPath.get(p.toLowerCase()) : undefined); if (twin !== undefined) { // Ours wins only when its words changed here too; a rename alone takes theirs. const ours = blobOid(twin.body) !== twin.synced_oid && twin.body !== body; if (ours) keepCopy(twin, body); changed.push( db .query<Note, [string, string, string, number]>( ours ? "UPDATE note SET synced_path = ?2, synced_oid = ?3 WHERE id = ?4 RETURNING *" : `UPDATE note SET body = ?1, synced_path = ?2, synced_oid = ?3, author_name = 'the repository', updated_at = ${NOW} WHERE id = ?4 RETURNING *`, ) .get(body, p, entry.oid, twin.id)!, ); if (!ours) pulled++; continue; } // A note here moved onto this name while the repository wrote its own // file there: keep theirs beside it. const squatter = byPath.get(p.toLowerCase()); if (squatter !== undefined) { keepCopy(squatter, body); continue; } // A file new to us: a note, titled by its name (or, when another // folder already has that name, by its name and a number). const title = freeTitle(db, vault, titleOfPath(p).replace(/[\[\]|#^\\]/g, " ").trim() || "Untitled"); changed.push( db .query<Note, [string, string, string, string]>( "INSERT INTO note (title, body, author_id, author_name, path, synced_path, synced_oid) VALUES (?1, ?2, ?3, 'the repository', ?4, ?4, ?5) RETURNING *", ) .get(title, body, vault, p, entry.oid)!, ); pulled++; } // Files the repository no longer has: gone here too, unless edited here // since — then they are written back. for (const n of notes) { if (n.synced_path === null || files.has(n.synced_path)) continue; if (unsynced(n)) db.query("UPDATE note SET synced_path = NULL, synced_oid = NULL WHERE id = ?1").run(n.id); else { db.query("DELETE FROM note WHERE id = ?1").run(n.id); removed.push(n.id); pulled++; } } for (const p of gone) if (!files.has(p)) db.query("DELETE FROM space_gone WHERE space_id = ?1 AND path = ?2").run(space.id, p); db.query("UPDATE space SET head = ?1 WHERE id = ?2").run(head, space.id); })(); for (const n of changed) tellSpace(db, space.id, "note.changed", { id: n.id, title: n.title, updated_at: n.updated_at }); for (const id of removed) tellSpace(db, space.id, "note.removed", { id }); return { pulled, copies }; } /** Commit whatever was written here since the repository last heard. */ async function push(db: Database, forge: Forge, space: Space): Promise<{ pushed: number; commit?: string }> { const notes = notesOf(db, space).filter(unsynced); const gone = db .query<{ path: string }, [number]>("SELECT path FROM space_gone WHERE space_id = ?1") .all(space.id) .map((g) => g.path); if (notes.length === 0 && gone.length === 0) return { pushed: 0 }; const writes = new Map<string, string | null>(); for (const n of notes) writes.set(n.path!, n.body); // A rename or a move leaves its old file behind; a removal takes its file. for (const n of notes) if (n.synced_path !== null && n.synced_path !== n.path && !writes.has(n.synced_path)) writes.set(n.synced_path, null); for (const p of gone) if (!writes.has(p)) writes.set(p, null); const files = [...writes].map(([path, content]) => ({ path, content })); const written = notes.map((n) => n.title); const who = [...new Set(notes.map((n) => n.author_name))].filter((n) => n !== "the repository"); const subject = written.length === 0 ? `notes: remove ${gone.length} ${gone.length === 1 ? "note" : "notes"}` : `notes: ${written.slice(0, 3).join(", ")}${written.length > 3 ? ` and ${written.length - 3} more` : ""}${gone.length > 0 ? `, removed ${gone.length}` : ""}`; const message = `${subject}${who.length > 0 ? `\n\nWritten in notes by ${who.join(", ")}.` : ""}`; const commit = await forge.commit(space.repo, space.branch, message, files); db.transaction(() => { for (const n of notes) db.query("UPDATE note SET synced_path = ?1, synced_oid = ?2 WHERE id = ?3").run(n.path, blobOid(n.body), n.id); for (const p of gone) db.query("DELETE FROM space_gone WHERE space_id = ?1 AND path = ?2").run(space.id, p); })(); // The head stays where it was read: someone may have pushed just before // this commit landed on top of theirs, and the next read must see it. // Files of ours it finds unchanged (their blob is the one we wrote). return { pushed: files.length, commit }; } const running = new Map<number, Promise<Report>>(); /** One pass for one space: read, then write. Passes for a space never overlap. */ export function syncSpace(db: Database, forge: Forge, id: number): Promise<Report> { const before = running.get(id) ?? Promise.resolve(undefined); const pass = before .catch(() => undefined) .then(async (): Promise<Report> => { const space = findSpace(db, id); if (space === undefined) return { pulled: 0, pushed: 0, copies: 0 }; try { const head = await forge.head(space.repo, space.branch); const read = await pull(db, forge, space, head); const wrote = await push(db, forge, findSpace(db, id) ?? space); db.query(`UPDATE space SET synced_at = ${NOW}, sync_error = NULL WHERE id = ?1`).run(id); if (read.pulled + read.copies + wrote.pushed > 0) tellSpace(db, id, "space.synced", { id }); return { ...read, pushed: wrote.pushed, ...(wrote.commit === undefined ? {} : { commit: wrote.commit }) }; } catch (e) { const message = e instanceof Error ? e.message : String(e); db.query(`UPDATE space SET synced_at = ${NOW}, sync_error = ?1 WHERE id = ?2`).run(message.slice(0, 500), id); tellSpace(db, id, "space.synced", { id }); throw e; } }); running.set(id, pass); void pass.finally(() => { if (running.get(id) === pass) running.delete(id); }).catch(() => {}); return pass; } export const WAIT_MS = 4_000; export const MAX_WAIT_MS = 20_000; export const EVERY_MS = 60_000; /** * Keep every space in step from now on: a pass a moment after each edit, * and one a minute for whatever was pushed to the repository. Returns the * way to stop. */ export function startSync(db: Database, forge: Forge | undefined): () => void { if (forge === undefined) return () => {}; const timers = new Map<number, { timer: ReturnType<typeof setTimeout>; first: number }>(); const quiet = (id: number) => syncSpace(db, forge, id).catch(() => {}); whenEdited((id) => { const owed = timers.get(id); const first = owed?.first ?? Date.now(); if (owed !== undefined) clearTimeout(owed.timer); const wait = Math.max(0, Math.min(WAIT_MS, first + MAX_WAIT_MS - Date.now())); timers.set(id, { first, timer: setTimeout(() => { timers.delete(id); void quiet(id); }, wait), }); }); const all = () => { for (const { id } of db.query<{ id: number }, []>("SELECT id FROM space").all()) void quiet(id); }; const minute = setInterval(all, EVERY_MS); setTimeout(all, 1_000); return () => { clearInterval(minute); for (const t of timers.values()) clearTimeout(t.timer); whenEdited(() => {}); }; }