mkdir prometheus-agent-test && cd prometheus-agent-test
cdk init --language typescript

mkdir metrics-app && cd metrics-app
npm init -y
npm install --save prom-client

'use strict';

const http = require('http');
const server = http.createServer();

const client = require('prom-client');
const register = new client.Registry()

// 5秒間隔でメトリクスを取得する
client.collectDefaultMetrics({ register, timeout: 5 * 1000 });

server.on('request', async function(req, res) {
    // /metrics にアクセスしたら、Prometheus のレポートを返す
    if (req.url === '/metrics') {
        res.setHeader('Content-Type', register.contentType)

        const metrics = await register.metrics()
        return res.end(metrics)
    } else {
        return res.writeHead(404, {'Content-Type' : 'text/plain'});
    }
});

server.listen(8080);

FROM public.ecr.aws/docker/library/node:16-alpine3.12 AS builder

EXPOSE 8080
WORKDIR /usr/src/app

COPY package*.json ./
RUN npm install --max-old-space-size=4096

COPY . .

CMD [ "node", "index.js" ]

docker build -t prometheus-agent-test/metrics-app .
docker run -p 8080:8080 prometheus-agent-test/metrics-app:latest

mkdir prometheus-agent && cd prometheus-agent

global:
  scrape_interval: 5s
  external_labels:
    monitor: 'prometheus'

scrape_configs:
  - job_name: 'prometheus-agent-test'
    static_configs:
      - targets: ['localhost:8080']
        labels:
          # デフォルトの localhost:8080 がインスタンスとして利用されると、
          # メトリクスの判別がしづらくなるため ECS Task の ID を利用する
          instance: '__TASK_ID__'

remote_write:
  # AMP ワークスペース作成時に控えておいた、
  # `エンドポイント - リモート書き込み URL` を設定する箇所
  - url: '__REMOTE_WRITE_URL__'
    sigv4:
      region: ap-northeast-1
    queue_config:
      max_samples_per_send: 1000
      max_shards: 200
      capacity: 2500

#!/bin/sh

while [ -z "$taskId" ]
do
  # ECS Fargate で起動したタスク ID を取得する
  taskId=$(curl --silent ${ECS_CONTAINER_METADATA_URI}/task | jq -r '.TaskARN | split("/") | .[-1]')

  echo "waiting..."
  sleep 1
done

echo "taskId: ${taskId}"
echo "remoteWriteUrl: ${REMOTE_WRITE_URL}"

# タスク ID `taskId` および、環境変数 `REMOTE_WRITE_URL` で、
# Prometheus のテンプレートファイル `prometheus.tmpl.yml` の内容を書き換え、
# その結果を `/etc/prometheus/prometheus.yml` に出力する
cat /etc/prometheus/prometheus.tmpl.yml | \
    sed "s/__TASK_ID__/${taskId}/g" | \
    sed "s>__REMOTE_WRITE_URL__>${REMOTE_WRITE_URL}>g" > /etc/prometheus/prometheus.yml

# --enable-feature=agent で Prometheus を Agent モードで起動する
# Prometheus のコンフィグファイルには上記で出力した `/etc/prometheus/prometheus.yml` を利用する
/usr/local/bin/prometheus \
    --enable-feature=agent \
    --config.file=/etc/prometheus/prometheus.yml \
    --web.console.libraries=/etc/prometheus/console_libraries \
    --web.console.templates=/etc/prometheus/consoles

FROM --platform=arm64 alpine:3.15

ADD prometheus.tmpl.yml /etc/prometheus/

RUN apk add --update --no-cache jq sed curl

# ARM64 で動作する Prometheus v2.32.1 を curl でダウンロード展開する
RUN curl -sL -O https://github.com/prometheus/prometheus/releases/download/v2.32.1/prometheus-2.32.1.linux-arm64.tar.gz
RUN tar -zxvf prometheus-2.32.1.linux-arm64.tar.gz && rm prometheus-2.32.1.linux-arm64.tar.gz

# `prometheus` コマンドを `/usr/local/bin/prometheus` に移動する
RUN mv prometheus-2.32.1.linux-arm64/prometheus /usr/local/bin/prometheus

COPY ./docker-entrypoint.sh /
RUN chmod +x /docker-entrypoint.sh
CMD ["/docker-entrypoint.sh"]

import { Construct } from 'constructs';
import {
  Stack,
  StackProps,
  aws_ecs as ecs,
  aws_logs as logs,
  aws_aps as aps,
  aws_ecs_patterns as ecs_patterns,
  aws_iam as iam,
  aws_elasticloadbalancingv2 as elbv2,
  Duration,
  CfnOutput,
} from 'aws-cdk-lib';

export class PrometheusAgentTestStack extends Stack {
  constructor(scope: Construct, id: string, props?: StackProps) {
    super(scope, id, props);

    // Node.js アプリに ecs_patterns.ApplicationLoadBalancedFargateService を利用して ALB 経由でアクセス可能にする
    const projectName = 'prometheus-agent-test';
    const fargateService = new ecs_patterns.ApplicationLoadBalancedFargateService(this, `${projectName}-fargate-service`, {
      serviceName: `${projectName}-fargate-service`,
      cpu: 256,
      desiredCount: 3,
      listenerPort: 80,
      taskImageOptions: {
        family: `${projectName}-taskdef`,
        image: ecs.ContainerImage.fromAsset('metrics-app'),
        containerPort: 8080,
        logDriver: ecs.LogDrivers.awsLogs({
          streamPrefix: `/${projectName}/metrics-app`,
          logRetention: logs.RetentionDays.ONE_DAY
        })
      },
      cluster: new ecs.Cluster(this, `${projectName}-cluster`, {
        clusterName: `${projectName}-cluster`
      }),
      memoryLimitMiB: 512,
    });
    fargateService.targetGroup.configureHealthCheck({
      path: "/metrics",
      timeout: Duration.seconds(8),
      interval: Duration.seconds(10),
      healthyThresholdCount: 2,
      unhealthyThresholdCount: 4,
      healthyHttpCodes: "200",
    });

    // 本質ではないが、Gravition2 で動作させたいために RuntimePlatform のプロパティを上書きしている
    const fargateServiceTaskdef = fargateService.taskDefinition.node.defaultChild as ecs.CfnTaskDefinition
    fargateServiceTaskdef.addPropertyOverride('RuntimePlatform', {
      CpuArchitecture: "ARM64",
      OperatingSystemFamily: "LINUX"
    });

    // AMP への書き込み権限を付与する
    fargateService.taskDefinition.taskRole.addManagedPolicy(
      iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonPrometheusRemoteWriteAccess')
    )

    // (2021/12/18) Amazon Managed Service for Prometheus のワークスペースを作成して、Prometheus の remote-write URL を取得する
    const apsWorkspace = new aps.CfnWorkspace(this, `${projectName}-prom-workspace`, {
      alias: `${projectName}-prom-workspace`,
    });
    const apsWorkspaceRemoteUrl = `${apsWorkspace.attrPrometheusEndpoint}api/v1/remote_write`;

    // (2021/12/18) 本記事で頻出する "エンドポイント - リモート書き込み URL" をコンソールに出力する
    new CfnOutput(this, 'prom-remote-write-url', {
      value: apsWorkspaceRemoteUrl,
      description: 'Prometheus Workspace の remote-write URL',
      exportName: 'PromRemoteWriteURL',
    });

    // AMP へメトリクス情報を送信するための Prometheus Agent コンテナを追加する
    const containerName = `${projectName}-prometheus-agent`
    fargateService.taskDefinition.addContainer(containerName, {
      containerName,
      image: ecs.ContainerImage.fromAsset('prometheus-agent'),
      memoryReservationMiB: 128,
      environment: {
        // (2021/12/18) CDK 経由で作成した Prometheus の remote-write URL を設定する
        REMOTE_WRITE_URL: apsWorkspaceRemoteUrl
      },
      logging: new ecs.AwsLogDriver({
        streamPrefix: `/${projectName}/prometheus-agent`,
        logRetention: logs.RetentionDays.ONE_DAY,
      })
    });

    // Grafana のタスク定義を作成する
    const grafanaDashboardTaskDefinition = new ecs.FargateTaskDefinition(this, `${projectName}-grafana-taskdef`, {
      family: `${projectName}-grafana-taskdef`
    });
    // Grafana のタスクが Prometheus Query を叩けるように権限付与する
    grafanaDashboardTaskDefinition.taskRole.addManagedPolicy(
      iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonPrometheusQueryAccess')
    )

    // Grafana のコンテナを追加する。パスプレフィクスには dashboard を設定する
    const grafanaDashboardContainerName = `${projectName}-grafana-dashboard`
    grafanaDashboardTaskDefinition.addContainer(grafanaDashboardContainerName, {
      containerName: grafanaDashboardContainerName,
      image: ecs.ContainerImage.fromRegistry('public.ecr.aws/ubuntu/grafana'),
      environment: {
        AWS_SDK_LOAD_CONFIG: 'true',
        GF_AUTH_SIGV4_AUTH_ENABLED: 'true',
        GF_SERVER_SERVE_FROM_SUB_PATH: 'true',
        GF_SERVER_ROOT_URL: '%(protocol)s://%(domain)s/dashboard'
      },
      portMappings: [{ containerPort: 3000 }],
      memoryLimitMiB: 512,
      logging: new ecs.AwsLogDriver({
        streamPrefix: `/${projectName}/grafana-dashboard`,
        logRetention: logs.RetentionDays.ONE_DAY,
      })
    });

    const grafanaDashboardServiceName = `${projectName}-grafana-dashboard-service`
    const grafanaDashboardService = new ecs.FargateService(this, grafanaDashboardServiceName, {
      serviceName: grafanaDashboardServiceName,
      cluster: fargateService.cluster,
      taskDefinition: grafanaDashboardTaskDefinition,
      desiredCount: 1
    });

    // Grafana のタスクを ALB のターゲットグループに紐づける
    fargateService.listener.addTargets(`${projectName}-grafana-dashboard-target`, {
      priority: 1,
      conditions: [
        elbv2.ListenerCondition.pathPatterns(['/dashboard/*']),
      ],
      healthCheck: {
        path: '/dashboard/login',
        interval: Duration.seconds(10),
        timeout: Duration.seconds(8),
        healthyThresholdCount: 2,
        unhealthyThresholdCount: 3,
        healthyHttpCodes: "200"
      },
      port: 3000,
      protocol: elbv2.ApplicationProtocol.HTTP,
      targets: [grafanaDashboardService],
    });
  }
}

echo 'GET http://<識別子>.ap-northeast-1.elb.amazonaws.com/metrics' | vegeta attack -duration=5s | vegeta report