mirror of
https://github.com/elder-plinius/G0DM0D3.git
synced 2026-11-16 07:02:25 +00:00
Replace request.text() with a streaming reader that cancels at 512 KiB — prevents a headerless or chunked-encoding request from buffering an unbounded payload into Worker memory. No WAF rule dependency. Also updates the file header and wrangler.toml comments to describe the current in-memory fallback (not the old direct-commit behavior). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
567 lines
19 KiB
TypeScript
567 lines
19 KiB
TypeScript
/**
|
|
* G0DM0D3 Telemetry Proxy — Cloudflare Pages Function
|
|
*
|
|
* Receives metadata events from the frontend and buffers them in
|
|
* Cloudflare KV. When enough batches accumulate (or enough time
|
|
* passes), pending batches are flushed as JSONL commits to
|
|
* HuggingFace. This prevents the 10k+ commit buildup that
|
|
* happens with per-request commits.
|
|
*
|
|
* URL: POST /api/telemetry
|
|
*
|
|
* Setup (Cloudflare Pages Dashboard → Settings → Environment Variables):
|
|
* HF_TOKEN — HuggingFace write token (hf_...)
|
|
* HF_DATASET_REPO — Target dataset repo (e.g. "pliny-the-prompter/g0dm0d3")
|
|
* TELEMETRY_FLUSH_SECRET — Bearer token for authenticated cron flush
|
|
*
|
|
* KV Binding (Cloudflare Pages Dashboard → Settings → Bindings):
|
|
* TELEMETRY_KV — KV namespace for buffering
|
|
*
|
|
* Without TELEMETRY_KV bound, telemetry accumulates in an
|
|
* in-memory isolate buffer and flushes to HF in batches.
|
|
* KV is preferred (survives isolate eviction) but not required.
|
|
*
|
|
* Cloudflare-level protections (configure in Dashboard → Security):
|
|
* - WAF rule: block POST /api/telemetry with Content-Length > 524288
|
|
* - Rate limiting rule: 20 req/min per IP to /api/telemetry
|
|
* - Restrict Origin header to godmod3.ai (optional)
|
|
* - Monthly usage alert on Workers & KV (paid plan required at scale)
|
|
*/
|
|
|
|
interface Env {
|
|
HF_TOKEN: string
|
|
HF_DATASET_REPO: string
|
|
HF_DATASET_BRANCH?: string
|
|
TELEMETRY_KV?: KVNamespace
|
|
TELEMETRY_FLUSH_SECRET?: string
|
|
}
|
|
|
|
interface TelemetryEvent {
|
|
type: string
|
|
timestamp: number
|
|
session_id: string
|
|
[key: string]: unknown
|
|
}
|
|
|
|
interface TelemetryPayload {
|
|
events: TelemetryEvent[]
|
|
}
|
|
|
|
const HF_API = 'https://huggingface.co/api'
|
|
|
|
const CORS_HEADERS = {
|
|
'Access-Control-Allow-Origin': '*',
|
|
'Access-Control-Allow-Methods': 'POST, OPTIONS',
|
|
'Access-Control-Allow-Headers': 'Content-Type',
|
|
'Access-Control-Max-Age': '86400',
|
|
}
|
|
|
|
// ── Size & batching limits ──────────────────────────────────────────
|
|
// Workers have 128 MiB memory. Keep individual requests small and
|
|
// cap the total data loaded during a flush.
|
|
const MAX_REQUEST_BYTES = 512 * 1024 // 512 KiB per inbound POST
|
|
const MAX_BATCH_EVENTS = 100 // events per request
|
|
const FLUSH_BATCH_COUNT = 50 // keys before triggering flush
|
|
const FLUSH_MAX_KEYS = 50 // max keys consumed per flush cycle
|
|
const FLUSH_MAX_BYTES = 5 * 1024 * 1024 // 5 MiB combined JSONL per HF commit
|
|
const FLUSH_AGE_MS = 30 * 60 * 1000 // 30 minutes
|
|
|
|
// ── In-memory isolate buffer (fallback when KV not bound) ───────────
|
|
// Same persistence model as the rate limiter: module-level state
|
|
// survives across requests within the same Worker isolate. We
|
|
// accumulate JSONL lines and flush to HF when the buffer is large
|
|
// or old enough. Isolate eviction loses buffered data — acceptable
|
|
// for best-effort telemetry, not acceptable for the commit explosion.
|
|
const MEM_FLUSH_COUNT = 50 // events before flush
|
|
const MEM_FLUSH_BYTES = 256 * 1024 // 256 KiB before flush
|
|
const MEM_FLUSH_AGE_MS = 15 * 60 * 1000 // 15 minutes before flush
|
|
const MEM_MAX_BYTES = 2 * 1024 * 1024 // 2 MiB hard cap — drop oldest if exceeded
|
|
const memBuffer: string[] = []
|
|
let memBufferBytes = 0
|
|
let memBufferFirstTs = 0
|
|
|
|
// ── Rate Limiter (dual: session + IP, in-memory per-isolate) ────────
|
|
const RATE_LIMIT_WINDOW_MS = 60_000
|
|
const RATE_LIMIT_MAX_SESSION = 10 // per session_id
|
|
const RATE_LIMIT_MAX_IP = 20 // per IP — catches session_id rotation
|
|
const rateLimitMap = new Map<string, number[]>()
|
|
|
|
function isRateLimited(key: string, max: number): boolean {
|
|
const now = Date.now()
|
|
let timestamps = rateLimitMap.get(key)
|
|
|
|
if (!timestamps) {
|
|
timestamps = []
|
|
rateLimitMap.set(key, timestamps)
|
|
}
|
|
|
|
const cutoff = now - RATE_LIMIT_WINDOW_MS
|
|
while (timestamps.length > 0 && timestamps[0] <= cutoff) {
|
|
timestamps.shift()
|
|
}
|
|
|
|
if (timestamps.length >= max) {
|
|
return true
|
|
}
|
|
|
|
timestamps.push(now)
|
|
|
|
// Prevent unbounded memory growth across isolate lifetime
|
|
if (rateLimitMap.size > 10_000) {
|
|
const it = rateLimitMap.keys()
|
|
for (let i = 0; i < 1_000; i++) {
|
|
const k = it.next()
|
|
if (k.done) break
|
|
rateLimitMap.delete(k.value)
|
|
}
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
function deriveSessionKey(event: TelemetryEvent): string {
|
|
const raw = JSON.stringify(event)
|
|
let h = 0
|
|
for (let i = 0; i < raw.length; i++) {
|
|
h = ((h << 5) - h + raw.charCodeAt(i)) | 0
|
|
}
|
|
return `__derived_${Math.abs(h).toString(36)}`
|
|
}
|
|
|
|
// ── Event Schema Validation ──────────────────────────────────────────
|
|
|
|
function validateEvent(event: unknown): event is TelemetryEvent {
|
|
if (typeof event !== 'object' || event === null || Array.isArray(event)) {
|
|
return false
|
|
}
|
|
const e = event as Record<string, unknown>
|
|
|
|
if (typeof e.type !== 'string' || e.type.length === 0) return false
|
|
if (typeof e.timestamp !== 'number' || !Number.isFinite(e.timestamp)) return false
|
|
if (typeof e.session_id !== 'string') return false
|
|
if (JSON.stringify(e).length > 65_536) return false
|
|
|
|
return true
|
|
}
|
|
|
|
// Handle CORS preflight
|
|
export const onRequestOptions: PagesFunction<Env> = async () => {
|
|
return new Response(null, { status: 204, headers: CORS_HEADERS })
|
|
}
|
|
|
|
// GET ?flush=force — authenticated cron trigger for guaranteed flush.
|
|
export const onRequestGet: PagesFunction<Env> = async (context) => {
|
|
const { request, env } = context
|
|
const url = new URL(request.url)
|
|
|
|
if (url.searchParams.get('flush') !== 'force') {
|
|
return new Response(null, { status: 204, headers: CORS_HEADERS })
|
|
}
|
|
|
|
if (!env.TELEMETRY_FLUSH_SECRET) {
|
|
return jsonResponse({ error: 'Flush not configured' }, 503)
|
|
}
|
|
|
|
const authHeader = request.headers.get('Authorization') || ''
|
|
const token = authHeader.replace(/^Bearer\s+/i, '')
|
|
if (!token || token !== env.TELEMETRY_FLUSH_SECRET) {
|
|
return jsonResponse({ error: 'Unauthorized' }, 401)
|
|
}
|
|
|
|
if (!env.TELEMETRY_KV || !env.HF_TOKEN || !env.HF_DATASET_REPO) {
|
|
return jsonResponse({ error: 'KV or HF not configured' }, 503)
|
|
}
|
|
|
|
const flushed = await flushKVToHF(env)
|
|
return jsonResponse({ flushed }, 200)
|
|
}
|
|
|
|
// Main handler
|
|
export const onRequestPost: PagesFunction<Env> = async (context) => {
|
|
const { request, env } = context
|
|
|
|
// ── Early size rejection (before reading body into memory) ────────
|
|
const contentLength = parseInt(request.headers.get('Content-Length') || '', 10)
|
|
if (contentLength > MAX_REQUEST_BYTES) {
|
|
return jsonResponse({ error: 'Request too large', limit: MAX_REQUEST_BYTES }, 413)
|
|
}
|
|
|
|
if (!env.HF_TOKEN || !env.HF_DATASET_REPO) {
|
|
const missing = []
|
|
if (!env.HF_TOKEN) missing.push('HF_TOKEN')
|
|
if (!env.HF_DATASET_REPO) missing.push('HF_DATASET_REPO')
|
|
console.error(`[Telemetry] Missing env vars: ${missing.join(', ')}`)
|
|
return jsonResponse({ error: `Telemetry not configured (missing: ${missing.join(', ')})` }, 503)
|
|
}
|
|
|
|
// ── IP-based rate limiting (catches session_id rotation attacks) ───
|
|
const clientIP = request.headers.get('CF-Connecting-IP') || request.headers.get('X-Forwarded-For') || 'unknown'
|
|
if (isRateLimited(`ip:${clientIP}`, RATE_LIMIT_MAX_IP)) {
|
|
return jsonResponse({ error: 'Rate limit exceeded — try again later' }, 429)
|
|
}
|
|
|
|
// ── Bounded body read (cancels stream at 512 KiB) ──────────────────
|
|
// Content-Length can be spoofed or absent (chunked), so we enforce
|
|
// the cap at the byte level while streaming — never buffer more than
|
|
// MAX_REQUEST_BYTES regardless of what the client claims.
|
|
let bodyText: string
|
|
try {
|
|
const reader = request.body?.getReader()
|
|
if (!reader) {
|
|
return jsonResponse({ error: 'No body' }, 400)
|
|
}
|
|
const chunks: Uint8Array[] = []
|
|
let totalBytes = 0
|
|
while (true) {
|
|
const { done, value } = await reader.read()
|
|
if (done) break
|
|
totalBytes += value.byteLength
|
|
if (totalBytes > MAX_REQUEST_BYTES) {
|
|
reader.cancel()
|
|
return jsonResponse({ error: 'Request too large', limit: MAX_REQUEST_BYTES }, 413)
|
|
}
|
|
chunks.push(value)
|
|
}
|
|
const merged = new Uint8Array(totalBytes)
|
|
let offset = 0
|
|
for (const c of chunks) { merged.set(c, offset); offset += c.byteLength }
|
|
bodyText = new TextDecoder().decode(merged)
|
|
} catch {
|
|
return jsonResponse({ error: 'Failed to read body' }, 400)
|
|
}
|
|
|
|
let payload: TelemetryPayload
|
|
try {
|
|
payload = JSON.parse(bodyText) as TelemetryPayload
|
|
} catch {
|
|
return jsonResponse({ error: 'Invalid JSON' }, 400)
|
|
}
|
|
|
|
if (!payload.events || !Array.isArray(payload.events) || payload.events.length === 0) {
|
|
return jsonResponse({ error: 'No events provided' }, 400)
|
|
}
|
|
|
|
const events = payload.events.slice(0, MAX_BATCH_EVENTS)
|
|
|
|
const invalid = events.filter(e => !validateEvent(e))
|
|
if (invalid.length > 0) {
|
|
return jsonResponse({ error: `${invalid.length} event(s) failed schema validation` }, 400)
|
|
}
|
|
|
|
// ── Session-based rate limiting ───────────────────────────────────
|
|
const firstEvent = events[0]
|
|
const sessionKey = firstEvent.session_id
|
|
? firstEvent.session_id
|
|
: deriveSessionKey(firstEvent)
|
|
|
|
if (isRateLimited(`session:${sessionKey}`, RATE_LIMIT_MAX_SESSION)) {
|
|
return jsonResponse({ error: 'Rate limit exceeded — try again later' }, 429)
|
|
}
|
|
|
|
const sanitized = events.map(stripPII)
|
|
const jsonl = sanitized.map(e => JSON.stringify(e)).join('\n')
|
|
|
|
// ── KV buffered path (preferred) ──────────────────────────────────
|
|
if (env.TELEMETRY_KV) {
|
|
const now = Date.now()
|
|
const kvKey = `batch:${now}:${shortHash(jsonl)}`
|
|
|
|
await env.TELEMETRY_KV.put(kvKey, jsonl, {
|
|
expirationTtl: 604800, // 7 days
|
|
})
|
|
|
|
// Check flush conditions
|
|
const pending = await env.TELEMETRY_KV.list({ prefix: 'batch:' })
|
|
const pendingCount = pending.keys.length
|
|
|
|
let shouldFlush = pendingCount >= FLUSH_BATCH_COUNT
|
|
|
|
if (!shouldFlush && pendingCount > 0) {
|
|
const oldestKey = pending.keys[0].name
|
|
const oldestTs = parseInt(oldestKey.split(':')[1], 10)
|
|
if (!isNaN(oldestTs) && (now - oldestTs) > FLUSH_AGE_MS) {
|
|
shouldFlush = true
|
|
}
|
|
}
|
|
|
|
if (shouldFlush) {
|
|
context.waitUntil(flushKVToHF(env))
|
|
}
|
|
|
|
return jsonResponse({
|
|
accepted: sanitized.length,
|
|
buffered: true,
|
|
pending: pendingCount,
|
|
flushing: shouldFlush,
|
|
}, 200)
|
|
}
|
|
|
|
// ── In-memory buffer fallback (KV not configured) ──────────────────
|
|
// Accumulates JSONL in the isolate's module scope and flushes to HF
|
|
// when the buffer is large or old enough. Zero config required.
|
|
// Data loss on isolate eviction is acceptable for telemetry.
|
|
const now = Date.now()
|
|
if (memBufferFirstTs === 0) memBufferFirstTs = now
|
|
|
|
// Drop oldest lines if hard cap would be exceeded
|
|
const lineBytes = new TextEncoder().encode(jsonl).byteLength
|
|
while (memBufferBytes + lineBytes > MEM_MAX_BYTES && memBuffer.length > 0) {
|
|
const dropped = memBuffer.shift()!
|
|
memBufferBytes -= new TextEncoder().encode(dropped).byteLength
|
|
}
|
|
|
|
memBuffer.push(jsonl)
|
|
memBufferBytes += lineBytes
|
|
|
|
const age = now - memBufferFirstTs
|
|
const shouldFlush =
|
|
memBuffer.length >= MEM_FLUSH_COUNT ||
|
|
memBufferBytes >= MEM_FLUSH_BYTES ||
|
|
age >= MEM_FLUSH_AGE_MS
|
|
|
|
if (shouldFlush) {
|
|
context.waitUntil(flushMemToHF(env))
|
|
}
|
|
|
|
return jsonResponse({
|
|
accepted: sanitized.length,
|
|
buffered: true,
|
|
pending: memBuffer.length,
|
|
flushing: shouldFlush,
|
|
storage: 'memory',
|
|
}, 200)
|
|
}
|
|
|
|
// ── KV Flush: read up to FLUSH_MAX_KEYS, cap at FLUSH_MAX_BYTES ────
|
|
|
|
async function flushKVToHF(env: Env): Promise<number> {
|
|
const kv = env.TELEMETRY_KV!
|
|
|
|
// Only list up to FLUSH_MAX_KEYS — leave the rest for the next cycle.
|
|
// This bounds both KV read ops and memory consumption.
|
|
const list = await kv.list({ prefix: 'batch:', limit: FLUSH_MAX_KEYS })
|
|
const keys = list.keys
|
|
|
|
if (keys.length === 0) return 0
|
|
|
|
// Read values with a cumulative size cap
|
|
const flushedKeys: { name: string }[] = []
|
|
const chunks: string[] = []
|
|
let totalBytes = 0
|
|
|
|
for (const key of keys) {
|
|
const value = await kv.get(key.name)
|
|
if (!value) {
|
|
flushedKeys.push(key) // empty key — still delete it
|
|
continue
|
|
}
|
|
|
|
const valueBytes = new TextEncoder().encode(value).byteLength
|
|
if (totalBytes + valueBytes > FLUSH_MAX_BYTES && chunks.length > 0) {
|
|
// Would exceed cap — stop here, remaining keys stay for next flush
|
|
break
|
|
}
|
|
|
|
chunks.push(value)
|
|
flushedKeys.push(key)
|
|
totalBytes += valueBytes
|
|
}
|
|
|
|
if (chunks.length === 0) {
|
|
// Only empty keys found — clean them up
|
|
if (flushedKeys.length > 0) {
|
|
await Promise.all(flushedKeys.map(key => kv.delete(key.name)))
|
|
}
|
|
return 0
|
|
}
|
|
|
|
const combined = chunks.join('\n')
|
|
const keyDigest = await sha256Hex(flushedKeys.map(k => k.name).sort().join(','))
|
|
const filePath = `telemetry/flush_${keyDigest}.jsonl`
|
|
const branch = env.HF_DATASET_BRANCH || 'main'
|
|
|
|
const ok = await commitToHF(env.HF_TOKEN, env.HF_DATASET_REPO, branch, filePath, combined)
|
|
|
|
if (ok) {
|
|
await Promise.all(flushedKeys.map(key => kv.delete(key.name)))
|
|
console.log(`[Telemetry] Flushed ${flushedKeys.length} batches (${(totalBytes / 1024).toFixed(0)} KiB) → ${filePath}`)
|
|
return flushedKeys.length
|
|
}
|
|
|
|
console.error(`[Telemetry] Flush failed — ${flushedKeys.length} batches remain in KV`)
|
|
return 0
|
|
}
|
|
|
|
// ── In-memory flush: drain buffer → one HF commit ──────────────────
|
|
|
|
async function flushMemToHF(env: Env): Promise<number> {
|
|
if (memBuffer.length === 0) return 0
|
|
|
|
// Snapshot and clear before the async commit so new requests
|
|
// accumulate into a fresh buffer, not into the one we're flushing.
|
|
const snapshot = memBuffer.splice(0, memBuffer.length)
|
|
const snapshotBytes = memBufferBytes
|
|
memBufferBytes = 0
|
|
memBufferFirstTs = 0
|
|
|
|
const combined = snapshot.join('\n')
|
|
const digest = await sha256Hex(combined)
|
|
const filePath = `telemetry/membuf_${digest}.jsonl`
|
|
const branch = env.HF_DATASET_BRANCH || 'main'
|
|
|
|
const ok = await commitToHF(env.HF_TOKEN, env.HF_DATASET_REPO, branch, filePath, combined)
|
|
|
|
if (ok) {
|
|
console.log(`[Telemetry] Memory flush: ${snapshot.length} batches (${(snapshotBytes / 1024).toFixed(0)} KiB) → ${filePath}`)
|
|
return snapshot.length
|
|
}
|
|
|
|
// Commit failed — push lines back so the next request retries.
|
|
// Prepend so ordering is roughly preserved.
|
|
memBuffer.unshift(...snapshot)
|
|
memBufferBytes += snapshotBytes
|
|
if (memBufferFirstTs === 0) memBufferFirstTs = Date.now()
|
|
console.error(`[Telemetry] Memory flush failed — ${snapshot.length} batches returned to buffer`)
|
|
return 0
|
|
}
|
|
|
|
// ── HuggingFace Hub Commit ───────────────────────────────────────────
|
|
|
|
async function commitToHF(
|
|
token: string,
|
|
repo: string,
|
|
branch: string,
|
|
filePath: string,
|
|
content: string,
|
|
): Promise<boolean> {
|
|
const url = `${HF_API}/datasets/${repo}/commit/${branch}`
|
|
|
|
const contentBase64 = utf8ToBase64(content)
|
|
const ndjson = [
|
|
JSON.stringify({ key: 'header', value: { summary: `[telemetry] ${filePath}` } }),
|
|
JSON.stringify({ key: 'file', value: { content: contentBase64, path: filePath, encoding: 'base64' } }),
|
|
].join('\n')
|
|
|
|
try {
|
|
const res = await fetch(url, {
|
|
method: 'POST',
|
|
headers: {
|
|
'Authorization': `Bearer ${token}`,
|
|
'Content-Type': 'application/x-ndjson',
|
|
},
|
|
body: ndjson,
|
|
})
|
|
|
|
if (!res.ok) {
|
|
const err = await res.text().catch(() => '')
|
|
if (res.status === 401 || res.status === 403) {
|
|
console.error(`[Telemetry] HF AUTH FAILED (${res.status}) — HF_TOKEN is invalid or lacks write access to "${repo}"`)
|
|
} else if (res.status === 404) {
|
|
console.error(`[Telemetry] HF REPO NOT FOUND (404) — "${repo}" does not exist on HuggingFace`)
|
|
} else {
|
|
console.error(`[Telemetry] HF commit failed (${res.status}): ${err.slice(0, 300)}`)
|
|
}
|
|
}
|
|
|
|
return res.ok
|
|
} catch (err) {
|
|
console.error(`[Telemetry] Network error:`, err)
|
|
return false
|
|
}
|
|
}
|
|
|
|
// ── Helpers ──────────────────────────────────────────────────────────
|
|
|
|
const ALLOWED_FIELDS = new Set<string>([
|
|
// Core envelope
|
|
'type', 'timestamp', 'session_id',
|
|
|
|
// Shared model/perf
|
|
'mode', 'model', 'duration_ms', 'response_length', 'success', 'error_type',
|
|
|
|
// Pipeline config
|
|
'pipeline', 'stm_modules', 'strategy', 'godmode', 'auto_retry',
|
|
'improve_mode', 'liquid_mode',
|
|
|
|
// AutoTune
|
|
'autotune', 'detected_context', 'confidence',
|
|
'autotune_context', 'autotune_confidence',
|
|
|
|
// Parseltongue
|
|
'parseltongue', 'triggers_found', 'technique', 'intensity',
|
|
'parseltongue_transform', 'parseltongue_transformed',
|
|
'encoding', 'encoding_rounds',
|
|
|
|
// ULTRAPLINIAN race
|
|
'ultraplinian', 'tier', 'models_queried', 'models_succeeded', 'models_refused',
|
|
'early_stop', 'early_threshold', 'winner_model', 'winner_score',
|
|
'winner_content_length', 'winner_duration_ms', 'winner_template',
|
|
'total_duration_ms', 'judge_model', 'model_results', 'all_scores',
|
|
'winner_combo', 'winner_source', 'race_result',
|
|
'combo', 'combos_attempted', 'combos_failed', 'combos_succeeded',
|
|
|
|
// Standard completion
|
|
'attempts', 'content_length', 'temperature', 'top_p',
|
|
'fallback_reason',
|
|
|
|
// Streaming / liquid
|
|
'stream', 'fast_stream', 'has_image',
|
|
'liquid_upgraded', 'liquid_upgrades',
|
|
|
|
// Harm classification
|
|
'classification',
|
|
|
|
// Structural context
|
|
'persona', 'prompt_length', 'conversation_depth', 'memory_count',
|
|
'no_log',
|
|
])
|
|
|
|
function stripPII(event: TelemetryEvent): TelemetryEvent {
|
|
const clean: Record<string, unknown> = {}
|
|
for (const key of Object.keys(event)) {
|
|
if (ALLOWED_FIELDS.has(key)) {
|
|
clean[key] = event[key]
|
|
}
|
|
}
|
|
return clean as TelemetryEvent
|
|
}
|
|
|
|
function shortHash(str: string): string {
|
|
let h = 0
|
|
for (let i = 0; i < str.length; i++) {
|
|
h = ((h << 5) - h + str.charCodeAt(i)) | 0
|
|
}
|
|
return Math.abs(h).toString(36).slice(0, 6)
|
|
}
|
|
|
|
async function sha256Hex(str: string): Promise<string> {
|
|
const data = new TextEncoder().encode(str)
|
|
const buf = await crypto.subtle.digest('SHA-256', data)
|
|
return [...new Uint8Array(buf)].map(b => b.toString(16).padStart(2, '0')).join('')
|
|
}
|
|
|
|
// Chunked UTF-8-safe base64: processes 32 KiB at a time to avoid
|
|
// creating a single enormous string via Array.from() on the full input.
|
|
function utf8ToBase64(content: string): string {
|
|
const bytes = new TextEncoder().encode(content)
|
|
const CHUNK = 32 * 1024
|
|
let binStr = ''
|
|
for (let i = 0; i < bytes.length; i += CHUNK) {
|
|
const slice = bytes.subarray(i, Math.min(i + CHUNK, bytes.length))
|
|
for (let j = 0; j < slice.length; j++) {
|
|
binStr += String.fromCharCode(slice[j])
|
|
}
|
|
}
|
|
return btoa(binStr)
|
|
}
|
|
|
|
function jsonResponse(data: unknown, status: number): Response {
|
|
return new Response(JSON.stringify(data), {
|
|
status,
|
|
headers: {
|
|
'Content-Type': 'application/json',
|
|
...CORS_HEADERS,
|
|
},
|
|
})
|
|
}
|