feat(regulations): compile drafts in a sandboxed LaTeX worker

.github/workflows/docker.yaml

@@ -32,3 +32,30 @@ jobs:
           platforms: linux/amd64,linux/arm64
           push: true
           tags: ghcr.io/polinetworkorg/backend:latest
+
+  latex-worker:
+    name: Build the LaTeX worker
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v5
+      - uses: docker/setup-qemu-action@v3
+      - uses: docker/setup-buildx-action@v3
+
+      - uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # FROM ghcr.io/polinetworkorg/legal-texlive (built by PoliNetworkOrg/legal): the package must grant this
+      # repository read access in its settings, since legal is private.
+      - name: Build the worker image
+        uses: docker/build-push-action@v5
+        with:
+          file: docker/latex-worker.Dockerfile
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ghcr.io/polinetworkorg/latex-worker:latest

docker/latex-worker.Dockerfile

@@ -0,0 +1,25 @@
+# The regulations editor's LaTeX compile worker (src/regulations/worker): a separate service, never the backend.
+# Built FROM the legal repository's TeX Live image, so previews use the same packages as the official PDFs.
+#   docker build -f docker/latex-worker.Dockerfile -t ghcr.io/polinetworkorg/latex-worker .
+# Run it as in polinetwork-cd: non-root, read-only root, tmpfs on /work and /tmp, no network egress, CPU/memory limits.
+ARG TEXLIVE_IMAGE=ghcr.io/polinetworkorg/legal-texlive:latest
+
+FROM oven/bun:1-debian AS build
+WORKDIR /build
+COPY package.json bun.lock ./
+RUN bun install --frozen-lockfile
+COPY tsconfig.base.json tsconfig.json ./
+COPY src ./src
+RUN bun build ./src/regulations/worker/server.ts --target bun --production --outfile /build/worker.js
+
+FROM oven/bun:1-debian AS bun
+
+FROM ${TEXLIVE_IMAGE}
+USER root
+COPY --from=bun /usr/local/bin/bun /usr/local/bin/bun
+COPY --from=build /build/worker.js /app/worker.js
+RUN mkdir -p /work && chown 1000:1000 /work
+USER 1000
+ENV WORKER_PORT=4200 WORKER_WORK_DIR=/work WORKER_TMP_DIR=/tmp WORKER_CONCURRENCY=2 WORKER_TIMEOUT_MS=45000
+EXPOSE 4200
+ENTRYPOINT ["bun", "/app/worker.js"]

package.json

@@ -13,6 +13,7 @@
     "build:npm": "NODE_ENV=production tsup --config tsup.npm.config.ts",
     "email": "email dev --port 3012 --dir src/emails/templates",
     "start": "bun run ./dist/server.js",
+    "worker": "bun ./src/regulations/worker/server.ts",
     "test": "vitest run",
     "db:push": "drizzle-kit push",
     "db:generate": "drizzle-kit generate",

src/regulations/compile/assets.ts

@@ -0,0 +1,75 @@
+import { readdir, readFile } from "node:fs/promises"
+import { join, relative } from "node:path"
+import type { AssetBundle } from "./protocol"
+
+/** Build products and outputs: never part of the assets. */
+const BUILD_ARTIFACT = /\.(aux|log|out|toc|fls|fdb_latexmk|xdv|synctex\.gz|bbl|blg|lof|lot)$/
+
+/** Whether `path` matches a legal.json source glob (`*` within one path segment, `**` across segments). */
+export function matchesGlob(path: string, glob: string): boolean {
+  const pattern = glob
+    .split("**")
+    .map((part) =>
+      part
+        .replace(/[.+^${}()|[\]\\]/g, "\\$&")
+        .replace(/\*/g, "[^/]*")
+        .replace(/\?/g, "[^/]")
+    )
+    .join(".*")
+  return new RegExp(`^${pattern}$`).test(path)
+}
+
+export type DocumentLayout = {
+  /** Directory of the document, relative to the repository root (legal.json `dir`). */
+  dir: string
+  entry: string
+  sources: string[]
+  /** Release asset names: built outputs to leave out. */
+  outputs: Record<string, string>
+  /** legal.json `shared`: the class directory. */
+  shared: string
+}
+
+/** Files under `root`, relative to it, skipping dot files and dot directories. */
+export async function listTree(root: string): Promise<string[]> {
+  const out: string[] = []
+  const visit = async (dir: string) => {
+    for (const entry of await readdir(dir, { withFileTypes: true })) {
+      if (entry.name.startsWith(".")) continue
+      const full = join(dir, entry.name)
+      if (entry.isDirectory()) await visit(full)
+      else out.push(relative(root, full).split("\\").join("/"))
+    }
+  }
+  await visit(root)
+  return out.sort()
+}
+
+/**
+ * Splits a document directory of a legal checkout into editable sources (matching `sources`) and the asset bundle the
+ * compile needs: everything else in the directory (fonts, images, included PDFs) plus the shared class, which goes
+ * under `shared/` and is linked next to the entry file by the worker.
+ */
+export async function readDocument(
+  repoRoot: string,
+  layout: DocumentLayout
+): Promise<{ sources: Record<string, string>; assets: AssetBundle }> {
+  const docRoot = join(repoRoot, layout.dir)
+  const outputs = new Set([...Object.values(layout.outputs), layout.entry.replace(/\.tex$/, ".pdf")])
+  const sources: Record<string, string> = {}
+  const files: Record<string, string> = {}
+  for (const path of await listTree(docRoot)) {
+    if (layout.sources.some((glob) => matchesGlob(path, glob))) {
+      sources[path] = await readFile(join(docRoot, path), "utf8")
+      continue
+    }
+    if (BUILD_ARTIFACT.test(path) || outputs.has(path) || path.endsWith(".tex") || path === "latexmkrc") continue
+    files[path] = (await readFile(join(docRoot, path))).toString("base64")
+  }
+  for (const path of await listTree(join(repoRoot, layout.shared))) {
+    if (!path.endsWith(".cls") && !path.endsWith(".sty")) continue
+    if (path.includes("/")) continue
+    files[`shared/${path}`] = (await readFile(join(repoRoot, layout.shared, path))).toString("base64")
+  }
+  return { sources, assets: { version: 1, files } }
+}

src/regulations/compile/client.ts

@@ -0,0 +1,47 @@
+import type { AssetBundle, CompileRequest, CompileResponse } from "./protocol"
+
+export type WorkerClient = { compile: (request: CompileRequest) => Promise<CompileResponse> }
+
+/**
+ * HTTP client of the LaTeX worker. When the worker doesn't have an asset bundle yet (a new version, or a restarted
+ * pod with an empty tmpfs) it uploads it and tries again.
+ */
+export function createWorkerClient(options: {
+  url: string
+  secret?: string
+  loadAssets: (hash: string) => Promise<AssetBundle>
+  timeoutMs?: number
+}): WorkerClient {
+  const headers = {
+    "Content-Type": "application/json",
+    ...(options.secret ? { Authorization: `Bearer ${options.secret}` } : {}),
+  }
+  const base = options.url.replace(/\/$/, "")
+  const post = async (request: CompileRequest): Promise<CompileResponse> => {
+    const response = await fetch(`${base}/compile`, {
+      method: "POST",
+      headers,
+      body: JSON.stringify(request),
+      signal: AbortSignal.timeout(options.timeoutMs ?? 120_000),
+    })
+    if (!response.ok) throw new Error(`LaTeX worker answered ${response.status}`)
+    return (await response.json()) as CompileResponse
+  }
+  return {
+    async compile(request) {
+      const first = await post(request)
+      if (first.status !== "assets_missing") return first
+      const bundle = await options.loadAssets(request.assets)
+      const upload = await fetch(`${base}/assets/${request.assets}`, {
+        method: "PUT",
+        headers,
+        body: JSON.stringify(bundle),
+        signal: AbortSignal.timeout(60_000),
+      })
+      if (!upload.ok) throw new Error(`LaTeX worker refused the assets (${upload.status})`)
+      const second = await post(request)
+      if (second.status === "assets_missing") throw new Error("LaTeX worker lost the assets")
+      return second
+    },
+  }
+}

src/regulations/compile/protocol.ts

@@ -0,0 +1,71 @@
+/**
+ * Wire protocol between the compile callers (collab service, backend) and the LaTeX worker. The worker has no network
+ * egress and no database: everything it needs comes in the request, and the assets (fonts, the shared class) are
+ * uploaded once per bundle hash and cached.
+ */
+
+/** Paths relative to the document directory (legal.json `dir`). */
+export type CompileRequest = {
+  /** Warm workdir key: the draft id (`.aux` survives between compiles), or `version:<id>`. */
+  key: string
+  /** Source hash (see `sourceHash`), echoed back. */
+  hash: string
+  entry: string
+  /** Hash of the asset bundle the sources compile with. */
+  assets: string
+  files: Record<string, string>
+}
+
+export type CompileError = { file: string | null; line: number | null; message: string }
+
+export type CompileResult = {
+  ok: boolean
+  /** Base64 PDF when `ok`. */
+  pdf: string | null
+  /** Tail of the xelatex log. */
+  log: string
+  errors: CompileError[]
+  durationMs: number
+  runs: number
+}
+
+export type CompileResponse =
+  | ({ status: "done" } & CompileResult)
+  /** A newer request for the same key arrived while this one waited: latest wins. */
+  | { status: "superseded" }
+  /** Upload the bundle with `PUT /assets/<hash>`, then retry. */
+  | { status: "assets_missing" }
+
+/** Non-editable files the compile needs, by path relative to the document directory; content in base64. */
+export type AssetBundle = { version: 1; files: Record<string, string> }
+
+export const WORKER_SECRET_HEADER = "authorization"
+
+async function sha256Hex(data: string | Uint8Array): Promise<string> {
+  const bytes = typeof data === "string" ? new TextEncoder().encode(data) : data
+  const digest = await crypto.subtle.digest("SHA-256", bytes as Uint8Array<ArrayBuffer>)
+  return Buffer.from(digest).toString("hex")
+}
+
+function canonical(record: Record<string, string>): string {
+  return JSON.stringify(
+    Object.keys(record)
+      .sort()
+      .map((k) => [k, record[k]])
+  )
+}
+
+/** Identifies a compile: same document, entry, sources and assets give the same PDF. */
+export function sourceHash(input: { entry: string; files: Record<string, string>; assets: string }): Promise<string> {
+  return sha256Hex(`v1\n${input.entry}\n${input.assets}\n${canonical(input.files)}`)
+}
+
+export function bundleHash(bundle: AssetBundle): Promise<string> {
+  return sha256Hex(`v1\n${canonical(bundle.files)}`)
+}
+
+/** Rejects absolute paths, `..` and anything that could escape the workdir or hide as a dot file. */
+export function safeRelativePath(path: string): boolean {
+  if (!path || path.length > 300 || path.startsWith("/") || path.includes("\\") || path.includes("\0")) return false
+  return path.split("/").every((part) => part !== "" && part !== "." && part !== ".." && !part.startsWith("."))
+}

src/regulations/compile/queue.ts

@@ -0,0 +1,57 @@
+export type Scheduled<T> = { status: "done"; value: T } | { status: "superseded" }
+
+type Pending<T> = { run: () => Promise<T>; resolve: (r: Scheduled<T>) => void; reject: (e: unknown) => void }
+
+/**
+ * At most one job in flight per key and `concurrency` overall; a job waiting for its key is replaced by a newer one
+ * (latest wins), and the replaced caller is told so. Used for compiles: one per draft, a few in parallel.
+ */
+export function createLatestWinsQueue<T>(concurrency: number) {
+  const running = new Set<string>()
+  const pending = new Map<string, Pending<T>>()
+  /** Keys with a pending job, in arrival order, waiting for a free slot. */
+  const ready: string[] = []
+  let active = 0
+
+  const pump = () => {
+    for (let i = 0; i < ready.length && active < concurrency; ) {
+      const key = ready[i]
+      if (running.has(key)) {
+        i++
+        continue
+      }
+      ready.splice(i, 1)
+      const job = pending.get(key)
+      if (!job) continue
+      pending.delete(key)
+      running.add(key)
+      active++
+      job
+        .run()
+        .then(
+          (value) => job.resolve({ status: "done", value }),
+          (error) => job.reject(error)
+        )
+        .finally(() => {
+          running.delete(key)
+          active--
+          pump()
+        })
+    }
+  }
+
+  return {
+    schedule(key: string, run: () => Promise<T>): Promise<Scheduled<T>> {
+      return new Promise<Scheduled<T>>((resolve, reject) => {
+        const previous = pending.get(key)
+        if (previous) previous.resolve({ status: "superseded" })
+        else ready.push(key)
+        pending.set(key, { run, resolve, reject })
+        pump()
+      })
+    },
+    /** Whether a job for `key` is running or waiting. */
+    busy: (key: string) => running.has(key) || pending.has(key),
+    stats: () => ({ active, waiting: pending.size }),
+  }
+}