Automatic R2 backup and versioning

Automatically backup and version files in r2 buckets using cloudflare queues, notifications and a worker.

Setup

Create a worker

npm create cloudflare@latest

Create a queue

npx wrangler queues create my-r2-backup-queue

Create a bucket

npx wrangler r2 bucket create my-r2-backup-bucket-1
# or
npx wrangler r2 bucket create my-r2-backup-bucket-1 -s InfrequentAccess

Note that writing to an infrequent access bucket also charges for reads which is charged at $9/month minimum, even for 1 read.

Add to wrangler

[[queues.consumers]]
queue = "my-r2-backup-queue"
# Required: this should match the name of the queue created above
# If you misspell the name, you will receive an error when attempting to publish your Worker.
max_batch_size = 1 # optional: defaults to 10
max_batch_timeout = 15 # optional: defaults to 5 seconds

[[r2_buckets]]
binding = "R2_backup_bucket"
bucket_name = "my-r2-backup-bucket-1"

Create worker for backup

Using hono for fetch, but only the queue part is required. The entries in buckets json and Environment keys are added for each bucket you want to backup. See below for details about that.

const cfAccountId = '' // Set your cloudflare account id here.

type Environment = {
    readonly R2_BUCKET_1: R2Bucket
    readonly R2_BUCKET_2: R2Bucket
    readonly R2_BUCKET_3: R2Bucket
    // ...
    
    readonly R2_backup_bucket: R2Bucket
}
const buckets = [
    {
        binding: 'R2_BUCKET_1',
        bucket: 'BUCKET_NAME',
        key: 'my-bucket-1', // key in backup bucket
    },
    {
        binding: 'R2_BUCKET_2',
        bucket: 'BUCKET_NAME',
        key: 'my-bucket-2',
    },
    {
        binding: 'R2_BUCKET_3',
        bucket: 'BUCKET_NAME',
        key: 'my-bucket-3',
    },
]

const app = new Hono<{
    Bindings: Environment
}>()
app.get('/', (c) => {
    return c.text('Nothing here.')
})

export default {
    fetch: app.fetch,
    async queue(batch: MessageBatch<any>, env: Environment) {
        for (const message of batch.messages) {
            if(cfAccountId && message.body.account !== cfAccountId) {
                console.error('Account not found', message.body.account)
                continue;
            }
            const action = message.body.action
            if(action === 'DeleteObject') break;
            const bucketName = message.body.bucket
            const bucket = buckets.find((b) => b.bucket === bucketName)
            if(!bucket) {
                console.log('Bucket not found', bucketName)
                continue;
            }
            const object = message.body.object
            if(!object) {
                console.log('Object not found', bucketName)
                continue;
            }
            const timeKey = new Date(message.body.eventTime).toISOString().replace(/:/g, '-')
            const key = `${bucket.key}/${object.key}/${timeKey}`
            const eTag = object.eTag
            const r2Bucket = env[bucket.binding]
            if(!r2Bucket) {
                console.error('Misconfiguration', bucket.binding)
                continue
            }
            const obj = await r2Bucket.get(object.key)
            if(!obj) {
                console.log('Object not found', key)
                continue
            }
            if(obj?.etag !== eTag) {
                console.log('Mismatch', key)
                continue;
            }
            const res = await env.R2_backup_bucket.put(key, obj.body, {
                httpMetadata: obj.httpMetadata,
                customMetadata: obj.customMetadata,
                storageClass: "InfrequentAccess",
            })
            console.log('Backup', bucketName, object.key, res.key)
        }
    },
}

Add buckets for backup

Do the following for each bucket you want to backup.

Create notifications

npx wrangler r2 bucket notification create BUCKET_NAME --event-type object-create --queue ij-drive-r2-backup-version-1

Docs - https://developers.cloudflare.com/workers/wrangler/commands/#notification-create

To check the notifications -

npx wrangler r2 bucket notification list BUCKET_NAME

Add to wrangler

[[r2_buckets]]
binding = "R2_BUCKET_1"
bucket_name = "BUCKET_NAME"

Add to json in worker

{
    binding: "R2_BUCKET_1", // binding in wrangler.toml
    bucket: "BUCKET_NAME", // actual bucket name
    key: "bucket-name", // key(folder) in backup bucket
}