benjivers/notes Public Public: anyone can read this; nobody can change it.

/**

 * A shared folder and its repository, kept in step. The notes live here,

 * so the page is as quick as a personal vault; the repository is where

 * they are kept, so they are plain markdown files anyone can clone, push

 * to, or open in another editor.

 *

 * One pass (`syncSpace`) reads, then writes:

 *

 *   read   when the branch has moved since the last read, walk the folder

 *          and take every file that changed. A note nobody has touched

 *          here takes the repository's words; a note edited here *and*

 *          there keeps ours, and the repository's version is kept beside

 *          it as "<title> (repo copy)" — nothing is ever lost.

 *   write  whatever was written, renamed, moved or removed here since,

 *          as one commit.

 *

 * A note knows what the repository last had (synced_path and the git blob

 * of its words, synced_oid), so "changed here" is a comparison, never a

 * flag to forget. Commits go through the forge's API with a token

 * (NOTES_REPO_TOKEN): the app's own token reads only.

 *

 * Writes are owed, not immediate: a change waits a few seconds for the

 * next (typing is many saves), and a slow trickle still commits within

 * MAX_WAIT_MS. Every space is also read once a minute.

 */

import type { Database } from "bun:sqlite";

import { createHash } from "node:crypto";

import type { Note } from "./actions.ts";

import { BODY_MAX } from "./actions.ts";

import { findSpace, spaceVault, tellSpace, titleOfPath, whenEdited, type Space } from "./spaces.ts";

/** The git blob id of some words: what `git hash-object` says of the file. */

export function blobOid(text: string): string {

  const bytes = Buffer.from(text, "utf8");

  return createHash("sha1").update(`blob ${bytes.length}\0`).update(bytes).digest("hex");

}

export interface Entry {

  readonly name: string;

  readonly type: "blob" | "tree" | string;

  readonly oid: string;

  readonly size: number;

}

/** The forge, as much of it as a shared folder needs. */

export interface Forge {

  /** The commit at the tip of a branch; null when there is none yet. */

  head(repo: string, branch: string): Promise<string | null>;

  /** A folder's entries at a commit; null when it doesn't exist. */

  list(repo: string, ref: string, path: string): Promise<Entry[] | null>;

  /** A file's words at a commit. */

  read(repo: string, ref: string, path: string): Promise<string>;

  /** Write and remove files (`content: null`) as one commit on `branch`. */

  commit(

    repo: string,

    branch: string,

    message: string,

    files: { path: string; content: string | null }[],

  ): Promise<string>;

}

type Fetch = (input: string, init?: RequestInit) => Promise<Response>;

/** The forge's JSON API (`<forge>/api/v1`), as a token that may write the repositories. */

export function httpForge(api: string, token: string, fetch: Fetch = globalThis.fetch): Forge {

  const base = api.replace(/\/+$/, "");

  const headers = { authorization: `Bearer ${token}`, "content-type": "application/json" };

  const at = (repo: string, rest: string) => `${base}/repos/${repo}/${rest}`;

  const path = (p: string) => p.split("/").map(encodeURIComponent).join("/");

  const failed = async (r: Response, what: string): Promise<never> => {

    throw new Error(`${what}: HTTP ${r.status} ${(await r.text()).slice(0, 200)}`);

  };

  return {

    async head(repo, branch) {

      const r = await fetch(at(repo, `commits?ref=${encodeURIComponent(branch)}&limit=1`), { headers });

      if (r.status === 404) throw new Error(`No repository ${repo} — or the token can't see it.`);

      if (!r.ok) return failed(r, "read the branch");

      const log = (await r.json()) as { oid?: string }[];

      return log[0]?.oid ?? null;

    },

    async list(repo, ref, p) {

      const r = await fetch(at(repo, `contents${p === "" ? "" : `/${path(p)}`}?ref=${encodeURIComponent(ref)}`), {

        headers,

      });

      if (r.status === 404) return null;

      if (!r.ok) return failed(r, `list ${p || "/"}`);

      const answer = (await r.json()) as { type?: string; entries?: Entry[] };

      return answer.type === "dir" ? (answer.entries ?? []) : null;

    },

    async read(repo, ref, p) {

      const r = await fetch(at(repo, `contents/${path(p)}?ref=${encodeURIComponent(ref)}`), { headers });

      if (!r.ok) return failed(r, `read ${p}`);

      const answer = (await r.json()) as { encoding?: string; content?: string };

      const content = answer.content ?? "";

      return answer.encoding === "base64" ? Buffer.from(content, "base64").toString("utf8") : content;

    },

    async commit(repo, branch, message, files) {

      const r = await fetch(at(repo, "edits"), {

        method: "POST",

        headers,

        body: JSON.stringify({ branch, message, files }),

      });

      if (!r.ok) return failed(r, "commit");

      const answer = (await r.json()) as { commit?: { commitOid?: string; oid?: string } | string };

      const c = answer.commit;

      return typeof c === "string" ? c : (c?.commitOid ?? c?.oid ?? "");

    },

  };

}

/** Folders a vault never holds: git's, and editors' own settings (`.obsidian`, `.vscode`). */

const hidden = (name: string): boolean => name.startsWith(".") || name === "node_modules";

/** Every markdown file under `folder`, path → blob id. */

async function walk(forge: Forge, space: Space, ref: string): Promise<Map<string, Entry>> {

  const files = new Map<string, Entry>();

  const visit = async (dir: string, depth: number): Promise<void> => {

    if (depth > 12) return;

    const entries = await forge.list(space.repo, ref, dir);

    for (const e of entries ?? []) {

      if (hidden(e.name)) continue;

      const p = dir === "" ? e.name : `${dir}/${e.name}`;

      if (e.type === "tree") await visit(p, depth + 1);

      else if (e.type === "blob" && /\.md$/i.test(e.name) && e.size <= BODY_MAX) files.set(p, e);

    }

  };

  await visit(space.folder, 0);

  return files;

}

const NOW = "strftime('%Y-%m-%dT%H:%M:%fZ', 'now')";

const notesOf = (db: Database, space: Space): Note[] =>

  db.query<Note, [string]>("SELECT * FROM note WHERE author_id = ?1").all(spaceVault(space.id));

/** Whether the note has words, a name or a place the repository hasn't been given yet. */

export const unsynced = (n: Note): boolean =>

  n.path !== null && (n.synced_path === null || n.synced_path !== n.path || n.synced_oid !== blobOid(n.body));

/** A title nobody in the vault has yet: `base`, else `base 2`, `base 3`, … */

function freeTitle(db: Database, vault: string, base: string): string {

  const taken = (t: string) =>

    db

      .query<{ id: number }, [string, string]>("SELECT id FROM note WHERE author_id = ?1 AND title = ?2 COLLATE NOCASE")

      .get(vault, t) !== null;

  if (!taken(base)) return base;

  for (let n = 2; ; n++) if (!taken(`${base} ${n}`)) return `${base} ${n}`;

}

export interface Report {

  /** Notes taken from the repository: new, changed, removed. */

  readonly pulled: number;

  /** Files the commit wrote or removed. */

  readonly pushed: number;

  /** Repository versions kept beside a note edited in both places. */

  readonly copies: number;

  readonly commit?: string;

}

/** Read what the repository has that we don't — when the branch has moved. */

async function pull(db: Database, forge: Forge, space: Space, head: string | null): Promise<{ pulled: number; copies: number }> {

  if (head === null || head === space.head) return { pulled: 0, copies: 0 };

  const vault = spaceVault(space.id);

  const files = await walk(forge, space, head);

  const notes = notesOf(db, space);

  const bySynced = new Map(notes.filter((n) => n.synced_path !== null).map((n) => [n.synced_path!, n]));

  const byPath = new Map(notes.filter((n) => n.path !== null).map((n) => [n.path!.toLowerCase(), n]));

  const gone = new Set(

    db

      .query<{ path: string }, [number]>("SELECT path FROM space_gone WHERE space_id = ?1")

      .all(space.id)

      .map((g) => g.path),

  );

  // Read every changed file first, then write them all at once.

  const wanted = [...files].filter(([p, e]) => bySynced.get(p)?.synced_oid !== e.oid && !gone.has(p));

  const words = new Map<string, string>();

  for (const [p] of wanted) words.set(p, await forge.read(space.repo, head, p));

  let pulled = 0;

  let copies = 0;

  const changed: Note[] = [];

  const removed: number[] = [];

  db.transaction(() => {

    const keepCopy = (note: Note, body: string) => {

      const title = freeTitle(db, vault, `${note.title} (repo copy)`);

      const dir = note.path!.includes("/") ? note.path!.slice(0, note.path!.lastIndexOf("/") + 1) : "";

      changed.push(

        db

          .query<Note, [string, string, string, string]>(

            "INSERT INTO note (title, body, author_id, author_name, path) VALUES (?1, ?2, ?3, 'the repository', ?4) RETURNING *",

          )

          .get(title, body, vault, `${dir}${title}.md`)!,

      );

      copies++;

    };

    for (const [p, entry] of files) {

      if (gone.has(p)) continue;

      const known = bySynced.get(p);

      if (known !== undefined && known.synced_oid === entry.oid) continue;

      const body = words.get(p)!;

      // Made here and there under one name before either had seen the other.

      const twin = known ?? (byPath.get(p.toLowerCase())?.synced_path === null ? byPath.get(p.toLowerCase()) : undefined);

      if (twin !== undefined) {

        // Ours wins only when its words changed here too; a rename alone takes theirs.

        const ours = blobOid(twin.body) !== twin.synced_oid && twin.body !== body;

        if (ours) keepCopy(twin, body);

        changed.push(

          db

            .query<Note, [string, string, string, number]>(

              ours

                ? "UPDATE note SET synced_path = ?2, synced_oid = ?3 WHERE id = ?4 RETURNING *"

                : `UPDATE note SET body = ?1, synced_path = ?2, synced_oid = ?3, author_name = 'the repository', updated_at = ${NOW} WHERE id = ?4 RETURNING *`,

            )

            .get(body, p, entry.oid, twin.id)!,

        );

        if (!ours) pulled++;

        continue;

      }

      // A note here moved onto this name while the repository wrote its own

      // file there: keep theirs beside it.

      const squatter = byPath.get(p.toLowerCase());

      if (squatter !== undefined) {

        keepCopy(squatter, body);

        continue;

      }

      // A file new to us: a note, titled by its name (or, when another

      // folder already has that name, by its name and a number).

      const title = freeTitle(db, vault, titleOfPath(p).replace(/[\[\]|#^\\]/g, " ").trim() || "Untitled");

      changed.push(

        db

          .query<Note, [string, string, string, string]>(

            "INSERT INTO note (title, body, author_id, author_name, path, synced_path, synced_oid) VALUES (?1, ?2, ?3, 'the repository', ?4, ?4, ?5) RETURNING *",

          )

          .get(title, body, vault, p, entry.oid)!,

      );

      pulled++;

    }

    // Files the repository no longer has: gone here too, unless edited here

    // since — then they are written back.

    for (const n of notes) {

      if (n.synced_path === null || files.has(n.synced_path)) continue;

      if (unsynced(n)) db.query("UPDATE note SET synced_path = NULL, synced_oid = NULL WHERE id = ?1").run(n.id);

      else {

        db.query("DELETE FROM note WHERE id = ?1").run(n.id);

        removed.push(n.id);

        pulled++;

      }

    }

    for (const p of gone) if (!files.has(p)) db.query("DELETE FROM space_gone WHERE space_id = ?1 AND path = ?2").run(space.id, p);

    db.query("UPDATE space SET head = ?1 WHERE id = ?2").run(head, space.id);

  })();

  for (const n of changed) tellSpace(db, space.id, "note.changed", { id: n.id, title: n.title, updated_at: n.updated_at });

  for (const id of removed) tellSpace(db, space.id, "note.removed", { id });

  return { pulled, copies };

}

/** Commit whatever was written here since the repository last heard. */

async function push(db: Database, forge: Forge, space: Space): Promise<{ pushed: number; commit?: string }> {

  const notes = notesOf(db, space).filter(unsynced);

  const gone = db

    .query<{ path: string }, [number]>("SELECT path FROM space_gone WHERE space_id = ?1")

    .all(space.id)

    .map((g) => g.path);

  if (notes.length === 0 && gone.length === 0) return { pushed: 0 };

  const writes = new Map<string, string | null>();

  for (const n of notes) writes.set(n.path!, n.body);

  // A rename or a move leaves its old file behind; a removal takes its file.

  for (const n of notes) if (n.synced_path !== null && n.synced_path !== n.path && !writes.has(n.synced_path)) writes.set(n.synced_path, null);

  for (const p of gone) if (!writes.has(p)) writes.set(p, null);

  const files = [...writes].map(([path, content]) => ({ path, content }));

  const written = notes.map((n) => n.title);

  const who = [...new Set(notes.map((n) => n.author_name))].filter((n) => n !== "the repository");

  const subject =

    written.length === 0

      ? `notes: remove ${gone.length} ${gone.length === 1 ? "note" : "notes"}`

      : `notes: ${written.slice(0, 3).join(", ")}${written.length > 3 ? ` and ${written.length - 3} more` : ""}${gone.length > 0 ? `, removed ${gone.length}` : ""}`;

  const message = `${subject}${who.length > 0 ? `\n\nWritten in notes by ${who.join(", ")}.` : ""}`;

  const commit = await forge.commit(space.repo, space.branch, message, files);

  db.transaction(() => {

    for (const n of notes)

      db.query("UPDATE note SET synced_path = ?1, synced_oid = ?2 WHERE id = ?3").run(n.path, blobOid(n.body), n.id);

    for (const p of gone) db.query("DELETE FROM space_gone WHERE space_id = ?1 AND path = ?2").run(space.id, p);

  })();

  // The head stays where it was read: someone may have pushed just before

  // this commit landed on top of theirs, and the next read must see it.

  // Files of ours it finds unchanged (their blob is the one we wrote).

  return { pushed: files.length, commit };

}

const running = new Map<number, Promise<Report>>();

/** One pass for one space: read, then write. Passes for a space never overlap. */

export function syncSpace(db: Database, forge: Forge, id: number): Promise<Report> {

  const before = running.get(id) ?? Promise.resolve(undefined);

  const pass = before

    .catch(() => undefined)

    .then(async (): Promise<Report> => {

      const space = findSpace(db, id);

      if (space === undefined) return { pulled: 0, pushed: 0, copies: 0 };

      try {

        const head = await forge.head(space.repo, space.branch);

        const read = await pull(db, forge, space, head);

        const wrote = await push(db, forge, findSpace(db, id) ?? space);

        db.query(`UPDATE space SET synced_at = ${NOW}, sync_error = NULL WHERE id = ?1`).run(id);

        if (read.pulled + read.copies + wrote.pushed > 0) tellSpace(db, id, "space.synced", { id });

        return { ...read, pushed: wrote.pushed, ...(wrote.commit === undefined ? {} : { commit: wrote.commit }) };

      } catch (e) {

        const message = e instanceof Error ? e.message : String(e);

        db.query(`UPDATE space SET synced_at = ${NOW}, sync_error = ?1 WHERE id = ?2`).run(message.slice(0, 500), id);

        tellSpace(db, id, "space.synced", { id });

        throw e;

      }

    });

  running.set(id, pass);

  void pass.finally(() => {

    if (running.get(id) === pass) running.delete(id);

  }).catch(() => {});

  return pass;

}

export const WAIT_MS = 4_000;

export const MAX_WAIT_MS = 20_000;

export const EVERY_MS = 60_000;

/**

 * Keep every space in step from now on: a pass a moment after each edit,

 * and one a minute for whatever was pushed to the repository. Returns the

 * way to stop.

 */

export function startSync(db: Database, forge: Forge | undefined): () => void {

  if (forge === undefined) return () => {};

  const timers = new Map<number, { timer: ReturnType<typeof setTimeout>; first: number }>();

  const quiet = (id: number) => syncSpace(db, forge, id).catch(() => {});

  whenEdited((id) => {

    const owed = timers.get(id);

    const first = owed?.first ?? Date.now();

    if (owed !== undefined) clearTimeout(owed.timer);

    const wait = Math.max(0, Math.min(WAIT_MS, first + MAX_WAIT_MS - Date.now()));

    timers.set(id, {

      first,

      timer: setTimeout(() => {

        timers.delete(id);

        void quiet(id);

      }, wait),

    });

  });

  const all = () => {

    for (const { id } of db.query<{ id: number }, []>("SELECT id FROM space").all()) void quiet(id);

  };

  const minute = setInterval(all, EVERY_MS);

  setTimeout(all, 1_000);

  return () => {

    clearInterval(minute);

    for (const t of timers.values()) clearTimeout(t.timer);

    whenEdited(() => {});

  };

}