Deploying Kafka on Cloud Platforms

aws kafka create-cluster \
  --cluster-name my-kafka-cluster \
  --kafka-version 3.5.1 \
  --broker-node-group-info \
    "ClientSubnets=[us-east-1a,us-east-1b],InstanceType=kafka.m5.large" \
  --encryption-info \
    "EncryptionAtRestEnabled=true"

confluent kafka cluster create my-cluster \
  --cloud gcp \
  --region us-central1 \
  --type dedicated \
  --key my-api-key \
  --secret my-api-secret

confluent security iam role-binding create \
  --principal User:my-user \
  --role DeveloperReadAll \
  --resource-prefix my-cluster

const { EventHubClient } = require("@azure/event-hubs");

async function sendEvent() {
  const client = EventHubClient.createFromConnectionString(
    "Endpoint=sb://<namespace>.servicebus.windows.net/;SharedAccessKeyName=<key-name>;SharedAccessKey=<key>;EntityPath=<event-hub-name>"
  );

  const eventData = {
    body: JSON.stringify({ message: "Hello from Event Hubs!" }),
    properties: { source: "kafka-migration" }
  };

  await client.send(eventData);
  console.log("Event sent successfully");
}

sendEvent().catch(console.error);

const { EventHubClient } = require("@azure/event-hubs");

async function readEvents() {
  const client = EventHubClient.createFromConnectionString(
    "Endpoint=sb://<namespace>.servicebus.windows.net/;SharedAccessKeyName=<key-name>;SharedAccessKey=<key>;EntityPath=<event-hub-name>"
  );

  const consumer = client.createConsumer(
    "$Default",
    "0",
    EventHubClient.GetPartitionInfo("0")
  );

  const events = await consumer.receiveBatch(10, 5000);
  console.log("Received events:", events);
}

readEvents().catch(console.error);

gcloud beta kafka clusters create my-kafka-cluster \
    --project=my-project \
    --location=us-central1 \
    --zone=us-central1-a \
    --cluster-config=projects/my-project/locations/us-central1/clusters/my-kafka-cluster/config \
    --network=default \
    --subnetwork=default \
    --min-broker-count=3 \
    --max-broker-count=10

cloud:
  provider: aws
  region: us-west-2
instance:
  type: m5.large
storage:
  type: ebs
network:
  type: vpc

cloud:
  provider: gcp
  region: us-central1
instance:
  type: n1-standard-2
storage:
  type: ssd
network:
  type: vpc

cloud:
  provider: azure
  region: eastus
instance:
  type: Standard_D2s_v3
storage:
  type: managed
network:
  type: virtual_network

# Enable TLS for Kafka
listeners=SSL://localhost:9093
ssl.keystore.location=/path/to/keystore.jks
ssl.keystore.password=changeit
ssl.key.password=changeit
ssl.truststore.location=/path/to/truststore.jks
ssl.truststore.password=changeit

aws kafka update-broker-count \
    --cluster-arn arn:aws:kafka:us-east-1:123456789012:cluster/my-kafka-cluster/abc123 \
    --current-broker-count 3 \
    --target-broker-count 4

aws cloudwatch put-metric-alarm \
  --alarm-name "KafkaBrokerHighCPU" \
  --metric-name CPUUtilization \
  --namespace "AWS/EC2" \
  --statistic Average \
  --period 300 \
  --threshold 80 \
  --comparison-operator GreaterThanThreshold \
  --evaluation-periods 2 \
  --alarm-description "Triggers when Kafka broker CPU exceeds 80%" \
  --actions-enabled \
  --alarm-actions arn:aws:sns:us-east-1:123456789012:KafkaAlerts

broker.id=0
listeners=PLAINTEXT://:9092
advertised.listeners=PLAINTEXT://kafka-broker-0.example.com:9092
log.dirs=/var/lib/kafka/data
num.partitions=12
default.replication.factor=2
log.retention.hours=168
auto.create.topics.enable=true

# kafka-config.yaml
listeners: PLAINTEXT://localhost:9092,SSL://:9093
advertised.listeners: PLAINTEXT://kafka-internal.example.com:9092,SSL://kafka-external.example.com:9093
listener.security.protocol.map: PLAINTEXT:PLAINTEXT,SSL:SSL
inter.broker.listener.name: SSL
ssl.truststore.location: /etc/kafka/certs/truststore.jks
ssl.truststore.password: changeit
ssl.keystore.location: /etc/kafka/certs/keystore.jks
ssl.keystore.password: changeit
ssl.key.password: changeit