S3 Logs

Read mock nginx access logs from S3 through the AWS provider, parse each line with parse string patterns, and expose the result through rank serve.

Source

examples/s3-logs/main.rank

use aws::{ S3 }
use std::Env
use std::HTTP
use std::Runtime
use std::collections::{ count, filter, flatMap, length, map }
use std::string::{ isBlank, startsWith, split, trim }

LogDay = `2026-05-11` | `2026-05-12`
StatusClass = `2xx` | `3xx` | `4xx` | `5xx` | `other`

AccessLogFile = Object {
  format: `nginx-access`,
  server: string,
  day: LogDay,
  raw: string,
}

AccessLogEntry = Object {
  raw: string,
  clientIp: string,
  timestamp: string,
  method: string,
  target: string,
  protocol: string,
  status: string,
  statusClass: StatusClass,
  bytesSent: string,
  referrer: string,
  userAgent: string,
}

SysEnv = Object {
  S3_ENDPOINT?: string,
  ...: string,
}

HealthRoute = HTTP::Route {
  method: `GET`,
  path: `/health`
}

LogsRoute = HTTP::Route {
  method: `GET`,
  path: `/logs`,
  query: Object {
    day?: LogDay,
  }
}

Routes = HealthRoute | LogsRoute

log_bucket = `rank-example-logs`
default_log_day: LogDay = `2026-05-12`
available_log_days: [LogDay] = [`2026-05-11`, `2026-05-12`]

{ S3_ENDPOINT } = Env<SysEnv> {}
s3_endpoint = S3_ENDPOINT ?? `http://127.0.0.1:4566`

s3Client: aws::S3::ClientConfig = S3::createClient({
  region: `us-east-1`,
  endpoint: s3_endpoint,
  credentials: {
    accessKeyId: `test`,
    secretAccessKey: `test`,
  },
})

status_class_for = |status: string| -> StatusClass {
  return startsWith(status, `2`) ? `2xx`
    : startsWith(status, `3`) ? `3xx`
    : startsWith(status, `4`) ? `4xx`
    : startsWith(status, `5`) ? `5xx`
    : `other`
}

log_key_for = |day: LogDay| -> string {
  return match day {
    `2026-05-11` => `nginx/2026-05-11/access.json`,
    `2026-05-12` => `nginx/2026-05-12/access.json`,
  }
}

parse_access_entries = |line: string| -> [AccessLogEntry] {
  return match line {
    parse`{clientIp:string} - {_remoteUser:string} [{timestamp:string}] "{method:string} {target:string} HTTP/{protocol:string}" {status:string} {bytesSent:string} "{referrer:string}" "{userAgent:string}"` => [{
      raw: line,
      clientIp,
      timestamp,
      method,
      target,
      protocol,
      status,
      statusClass: status_class_for(status),
      bytesSent,
      referrer,
      userAgent,
    }],
    _ => [],
  }
}

list_logs = |day: LogDay| -> HTTP::Response {
  key = log_key_for(day)

  fetched: aws::S3::ObjectResult<AccessLogFile> = aws::S3::Object<AccessLogFile> {
    client: s3Client,
    bucket: log_bucket,
    key: key,
  }

  { object, metadata } = fetched

  non_blank_lines = object.raw
    |> split(`\n`)
    |> map(|line: string| trim(line))
    |> filter(|line: string| !isBlank(line))

  entries = non_blank_lines
    |> flatMap(|line: string| parse_access_entries(line))

  return {
    status: 200,
    body: {
      availableDays: available_log_days,
      source: {
        bucket: log_bucket,
        key: key,
        server: object.server,
        day: object.day,
        requestId: metadata.requestId,
      },
      summary: {
        totalLines: length(non_blank_lines),
        parsedLines: length(entries),
        droppedLines: length(non_blank_lines) - length(entries),
        okResponses: count(entries, |entry: AccessLogEntry| entry.statusClass == `2xx`),
        redirects: count(entries, |entry: AccessLogEntry| entry.statusClass == `3xx`),
        clientErrors: count(entries, |entry: AccessLogEntry| entry.statusClass == `4xx`),
        serverErrors: count(entries, |entry: AccessLogEntry| entry.statusClass == `5xx`),
      },
      entries: entries,
    }
  }
}

pub config = {
  defaultResponseFormat: `json`
}

pub main = |req: Runtime::ExecutionContext<Routes>| -> HTTP::Response {
  return match req {
    HealthRoute => {
      status: 200,
      body: {
        ok: true,
        service: `s3-logs-example`,
      },
    },
    LogsRoute => list_logs(req.query.day ?? default_log_day),
  }
}

examples/s3-logs/rank.toml

manifestVersion = 1

[package]
name = "s3-logs-example"
version = "0.1.0"
source = "."

[providers]
aws = { path = "../../packages/plugins/aws" }

[security]
allow-provider-capabilities = ["network"]
allow-env = ["S3_ENDPOINT"]

Output

{
  "availableDays": ["2026-05-11", "2026-05-12"],
  "source": {
    "bucket": "rank-example-logs",
    "key": "nginx/2026-05-12/access.json",
    "server": "edge-2",
    "day": "2026-05-12"
  },
  "summary": {
    "totalLines": 5,
    "parsedLines": 5,
    "droppedLines": 0,
    "okResponses": 1,
    "redirects": 0,
    "clientErrors": 2,
    "serverErrors": 2
  }
}

Key concepts

• aws::S3::Object<T> — reads the seeded object from S3 and validates the wrapper shape before any line parsing runs.
• parse string patterns — the log parser uses a single structured-text match arm instead of separate regex capture indexing.
• Pipe + flatMap — the raw text is split, trimmed, filtered, and parsed through one left-to-right collection flow.
• Runtime::ExecutionContext<Routes> — GET /health and GET /logs share one route union with typed query narrowing for day.
• Current S3 surface — the current AWS provider reads S3 objects as JSON only, so the seeded raw nginx text lives inside the raw field of a JSON object.

Run it

Start LocalStack from examples/s3-logs:

docker compose up

If another LocalStack setup is already using port 4566, choose a different host port:

LOCALSTACK_PORT=4567 docker compose up

Then start the Rank server from the repository root:

npm start -- serve examples/s3-logs --dev

If you changed the LocalStack port, pass the matching endpoint to the Rank server:

S3_ENDPOINT=http://127.0.0.1:4567 npm start -- serve examples/s3-logs --dev

Try it:

curl http://127.0.0.1:3000/health
curl http://127.0.0.1:3000/logs
curl 'http://127.0.0.1:3000/logs?day=2026-05-11'