claw-apply/lib/storage.mjs

/**
 * storage.mjs — Pluggable data storage (local disk or S3)
 *
 * When type is "local": reads/writes go to local disk (default).
 * When type is "s3": S3 is the primary store. No local files for data.
 *   - Versioned bucket means every write is recoverable.
 *   - In-memory cache in queue.mjs handles read performance.
 *
 * Config in settings.json:
 *   storage: { type: "s3", bucket: "claw-apply-data", region: "us-west-2" }
 *   storage: { type: "local" }  (default)
 */
import { readFileSync, writeFileSync, existsSync } from 'fs';
import { basename } from 'path';

let _s3Client = null;
let _config = { type: 'local' };

export function initStorage(settings) {
  _config = settings?.storage || { type: 'local' };
}

export function storageType() {
  return _config?.type || 'local';
}

function getS3Key(filePath) {
  return `data/${basename(filePath)}`;
}

async function getS3Client() {
  if (_s3Client) return _s3Client;
  const { S3Client: Client, PutObjectCommand, GetObjectCommand } = await import('@aws-sdk/client-s3');
  _s3Client = {
    client: new Client({ region: _config.region || 'us-west-2' }),
    PutObjectCommand,
    GetObjectCommand,
  };
  return _s3Client;
}

/**
 * Load JSON data. Source depends on storage type.
 */
export async function loadJSON(filePath, defaultValue = []) {
  if (_config.type === 's3') {
    try {
      const s3 = await getS3Client();
      const response = await s3.client.send(new s3.GetObjectCommand({
        Bucket: _config.bucket,
        Key: getS3Key(filePath),
      }));
      const body = await response.Body.transformToString();
      const parsed = JSON.parse(body);
      if (Array.isArray(defaultValue) && !Array.isArray(parsed)) {
        throw new Error(`Expected array, got ${typeof parsed}`);
      }
      return parsed;
    } catch (err) {
      if (err.name === 'NoSuchKey') return defaultValue;
      console.warn(`⚠️  S3 load failed for ${basename(filePath)}: ${err.message}`);
      return defaultValue;
    }
  }

  // Local storage
  if (!existsSync(filePath)) return defaultValue;
  try {
    const raw = readFileSync(filePath, 'utf8');
    const parsed = JSON.parse(raw);
    if (Array.isArray(defaultValue) && !Array.isArray(parsed)) {
      throw new Error(`Expected array, got ${typeof parsed}`);
    }
    return parsed;
  } catch (err) {
    console.warn(`⚠️  Local ${basename(filePath)} is corrupt: ${err.message}`);
    return defaultValue;
  }
}

/**
 * Save JSON data. Destination depends on storage type.
 * Validates data before writing to prevent corruption.
 */
export async function saveJSON(filePath, data) {
  if (typeof data === 'string' || data === null || data === undefined) {
    throw new Error(`Refusing to save ${typeof data} to ${basename(filePath)} — data corruption prevented`);
  }

  const body = JSON.stringify(data, null, 2);

  if (_config.type === 's3') {
    const s3 = await getS3Client();
    await s3.client.send(new s3.PutObjectCommand({
      Bucket: _config.bucket,
      Key: getS3Key(filePath),
      Body: body,
      ContentType: 'application/json',
    }));
    return;
  }

  // Local storage — atomic write
  const tmp = filePath + '.tmp';
  writeFileSync(tmp, body);
  const { renameSync } = await import('fs');
  renameSync(tmp, filePath);
}