DOCS: Add docs for self-managed Kafka cluster

content/en/integrations/kafka/docker-compose.yml

@@ -0,0 +1,72 @@
+version: '3.9'
+services:
+  zookeeper:
+    image: confluentinc/cp-zookeeper:6.0.2
+    container_name: zookeeper
+    hostname: zookeeper
+    ports:
+      - "2181:2181"
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+
+  kafka:
+    image: confluentinc/cp-kafka:6.0.2
+    container_name: kafka
+    hostname: kafka
+    restart: always
+    depends_on:
+      - zookeeper
+    ports:
+      - "9092:9092"
+      - "9101:9101"
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+      KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_JMX_PORT: 9101
+      KAFKA_JMX_HOSTNAME: localhost
+      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka:29092
+      CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
+      CONFLUENT_METRICS_ENABLE: 'true'
+      CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'
+      KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
+
+  kowl:
+    image: quay.io/cloudhut/kowl:v1.3.1
+    container_name: kowl
+    restart: always
+    ports:
+      - "8080:8080"
+    depends_on:
+      - kafka
+    environment:
+      - KAFKA_BROKERS=kafka:29092
+
+  localstack:
+    container_name: "${LOCALSTACK_DOCKER_NAME-localstack_main}"
+    image: localstack/localstack
+    network_mode: bridge
+    ports:
+      - "4566:4566"
+    depends_on:
+      - kafka
+      - kowl
+    environment:
+      - SERVICES=lambda,secretsmanager
+      - DEBUG=${DEBUG- }
+      - DATA_DIR=${DATA_DIR- }
+      - LAMBDA_EXECUTOR=${LAMBDA_EXECUTOR- }
+      - KINESIS_ERROR_PROBABILITY=${KINESIS_ERROR_PROBABILITY- }
+      - DOCKER_HOST=unix:///var/run/docker.sock
+      - HOST_TMP_FOLDER=${TMPDIR}
+    volumes:
+      - "${TMPDIR:-/tmp/localstack}:/tmp/localstack"
+      - "/var/run/docker.sock:/var/run/docker.sock"

content/en/integrations/kafka/index.md

@@ -0,0 +1,107 @@
+---
+title: "Self-managed Kafka cluster"
+tags: ["kafka", "self-managed"]
+categories: []
+weight: 11
+description: >
+  Using Localstack lambda with self-managed Kafka cluster
+---
+
+Localstack does not currently support AWS MSK out of the box, but you can run your own self-managed Kafka cluster and integrate it with your own applications.
+
+## Running self-managed Kafka
+
+You can find the [example Docker Compose](docker-compose.yml) file which contains a single-noded ZooKeeper and a Kafka cluster and a simple LocalStack setup as well as [Kowl](https://github.com/cloudhut/kowl), an Apache Kafka Web UI.
+
+1. Run Docker Compose:
+
+```sh
+$ docker-compose up -d
+```
+
+2. Create the lambda function:
+
+```sh
+$ awslocal lambda create-function \
+    --function-name fun1 \
+    --handler lambda.handler \
+    --runtime python3.8 \
+    --role r1 \
+    --zip-file fileb://lambda.zip
+{
+    "FunctionName": "fun1",
+    "FunctionArn": "arn:aws:lambda:us-east-1:000000000000:function:fun1",
+    "Runtime": "python3.8",
+    "Role": "r1",
+    "Handler": "lambda.handler",
+    "CodeSize": 294,
+    "Description": "",
+    "Timeout": 3,
+    "LastModified": "2021-05-19T02:01:06.617+0000",
+    "CodeSha256": "/GPsiNXaq4tBA4QpxPCwgpeVfP7j+1tTH6zdkJ3jiU4=",
+    "Version": "$LATEST",
+    "VpcConfig": {},
+    "TracingConfig": {
+        "Mode": "PassThrough"
+    },
+    "RevisionId": "d85469d2-8558-4d75-bc0e-5926f373e12c",
+    "State": "Active",
+    "LastUpdateStatus": "Successful",
+    "PackageType": "Zip"
+}
+```
+
+3. Create an example secret:
+
+```sh
+$ awslocal secretsmanager create-secret --name localstack
+{
+    "ARN": "arn:aws:secretsmanager:us-east-1:000000000000:secret:localstack-TDIuI",
+    "Name": "localstack",
+    "VersionId": "32bbb8e2-46ee-4322-b3d5-b6459d54513b"
+}
+```
+
+4. Create an example kafka topic:
+
+```sh
+$ docker exec -ti kafka kafka-topics --zookeeper zookeeper:2181 --create --replication-factor 1 --partitions 1 --topic t1
+Created topic t1.
+```
+
+5. Create the event source mapping to your local kafka cluster:
+
+```sh
+$ awslocal lambda create-event-source-mapping \
+    --topics t1 \
+    --source-access-configuration Type=SASL_SCRAM_512_AUTH,URI=arn:aws:secretsmanager:us-east-1:000000000000:secret:localstack-TDIuI \
+    --function-name arn:aws:lambda:us-east-1:000000000000:function:fun1 \
+    --self-managed-event-source '{"Endpoints":{"KAFKA_BOOTSTRAP_SERVERS":["localhost:9092"]}}'
+{
+    "UUID": "4a2b0ea6-960c-4847-8684-465876dd6dbd",
+    "BatchSize": 100,
+    "FunctionArn": "arn:aws:lambda:us-east-1:000000000000:function:fun1",
+    "LastModified": "2021-05-19T04:02:49+02:00",
+    "LastProcessingResult": "OK",
+    "State": "Enabled",
+    "StateTransitionReason": "User action",
+    "Topics": [
+        "t1"
+    ],
+    "SourceAccessConfigurations": [
+        {
+            "Type": "SASL_SCRAM_512_AUTH",
+            "URI": "arn:aws:secretsmanager:us-east-1:000000000000:secret:localstack-TDIuI"
+        }
+    ],
+    "SelfManagedEventSource": {
+        "Endpoints": {
+            "KAFKA_BOOTSTRAP_SERVERS": [
+                "localhost:9092"
+            ]
+        }
+    }
+}
+```
+
+6. Aditionally visit `http://localhost:8080` for Kowl's UI.