Post

kafka cluster in kuber

Posted Updated
By 4 min read
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232

apiVersion: apps/v1
kind: Deployment
metadata:
  name: kafka-1
spec:
  replicas: 1
  selector:
    matchLabels:
      app: kafka
      instance: kafka-1
  template:
    metadata:
      labels:
        app: kafka
        instance: kafka-1
    spec:
      containers:
      - name: kafka
        image: apache/kafka:3.7.1
        ports:
        - containerPort: 9092
        - containerPort: 9093
        env:
        - name: KAFKA_PROCESS_ROLES
          value: "broker,controller"
        - name: KAFKA_NODE_ID
          value: "1"
        - name: KAFKA_CONTROLLER_QUORUM_VOTERS
          value: "1@kafka-1:9093,2@kafka-2:9093,3@kafka-3:9093"
        - name: KAFKA_LISTENERS
          value: "PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093"
        - name: KAFKA_ADVERTISED_LISTENERS
          value: "PLAINTEXT://192.168.124.174:9092"
        - name: KAFKA_LOG_DIRS
          value: "/var/lib/kafka/data"
        - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP
          value: "PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT"
        - name: KAFKA_INTER_BROKER_LISTENER_NAME
          value: "PLAINTEXT"
        - name: KAFKA_AUTO_CREATE_TOPICS_ENABLE
          value: "false"
        - name: KAFKA_CONTROLLER_LISTENER_NAMES
          value: "CONTROLLER"
        - name: KAFKA_LOG_RETENTION_MS
          value: "300000"
        - name: KAFKA_MESSAGE_MAX_BYTES
          value: "2000000000"  # 2GB로 설정
        - name: KAFKA_REPLICA_FETCH_MAX_BYTES
          value: "2000000000"  # 2GB로 설정
        - name: KAFKA_MAX_PARTITION_FETCH_BYTES
          value: "2000000000"  # 2GB로 설정
        volumeMounts:
        - name: kafka-storage
          mountPath: /var/lib/kafka/data
      volumes:
      - name: kafka-storage
        emptyDir: {} # 파드 삭제되면 데이터도 날아감. Persistent Volum 쓰면 좋음.

---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: kafka-2
spec:
  replicas: 1
  selector:
    matchLabels:
      app: kafka
      instance: kafka-2
  template:
    metadata:
      labels:
        app: kafka
        instance: kafka-2
    spec:
      containers:
      - name: kafka
        image: apache/kafka:3.7.1
        ports:
        - containerPort: 9092
        - containerPort: 9093
        env:
        - name: KAFKA_PROCESS_ROLES
          value: "broker,controller"
        - name: KAFKA_NODE_ID
          value: "2"
        - name: KAFKA_CONTROLLER_QUORUM_VOTERS
          value: "1@kafka-1:9093,2@kafka-2:9093,3@kafka-3:9093"
        - name: KAFKA_LISTENERS
          value: "PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093"
        - name: KAFKA_ADVERTISED_LISTENERS
          value: "PLAINTEXT://192.168.124.173:9092"
        - name: KAFKA_LOG_DIRS
          value: "/var/lib/kafka/data"
        - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP
          value: "PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT"
        - name: KAFKA_INTER_BROKER_LISTENER_NAME
          value: "PLAINTEXT"
        - name: KAFKA_AUTO_CREATE_TOPICS_ENABLE
          value: "true"
        - name: KAFKA_CONTROLLER_LISTENER_NAMES
          value: "CONTROLLER"
        - name: KAFKA_LOG_RETENTION_MS
          value: "300000"
        - name: KAFKA_MESSAGE_MAX_BYTES
          value: "2000000000"  # 2GB로 설정
        - name: KAFKA_REPLICA_FETCH_MAX_BYTES
          value: "2000000000"  # 2GB로 설정
        - name: KAFKA_MAX_PARTITION_FETCH_BYTES
          value: "2000000000"  # 2GB로 설정
        volumeMounts:
        - name: kafka-storage
          mountPath: /var/lib/kafka/data
      volumes:
      - name: kafka-storage
        emptyDir: {}

---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: kafka-3
spec:
  replicas: 1
  selector:
    matchLabels:
      app: kafka
      instance: kafka-3
  template:
    metadata:
      labels:
        app: kafka
        instance: kafka-3
    spec:
      containers:
      - name: kafka
        image: apache/kafka:3.7.1
        ports:
        - containerPort: 9092
        - containerPort: 9093
        env:
        - name: KAFKA_PROCESS_ROLES
          value: "broker,controller"
        - name: KAFKA_NODE_ID
          value: "3"
        - name: KAFKA_CONTROLLER_QUORUM_VOTERS
          value: "1@kafka-1:9093,2@kafka-2:9093,3@kafka-3:9093"
        - name: KAFKA_LISTENERS
          value: "PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093"
        - name: KAFKA_ADVERTISED_LISTENERS
          value: "PLAINTEXT://192.168.78.213:9092"
        - name: KAFKA_LOG_DIRS
          value: "/var/lib/kafka/data"
        - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP
          value: "PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT"
        - name: KAFKA_INTER_BROKER_LISTENER_NAME
          value: "PLAINTEXT"
        - name: KAFKA_AUTO_CREATE_TOPICS_ENABLE
          value: "true"
        - name: KAFKA_CONTROLLER_LISTENER_NAMES
          value: "CONTROLLER"
        - name: KAFKA_LOG_RETENTION_MS
          value: "300000"
        - name: KAFKA_MESSAGE_MAX_BYTES
          value: "2000000000"  # 2GB로 설정
        - name: KAFKA_REPLICA_FETCH_MAX_BYTES
          value: "2000000000"  # 2GB로 설정
        - name: KAFKA_MAX_PARTITION_FETCH_BYTES
          value: "2000000000"  # 2GB로 설정
        volumeMounts:
        - name: kafka-storage
          mountPath: /var/lib/kafka/data
      volumes:
      - name: kafka-storage
        emptyDir: {}

---
apiVersion: v1
kind: Service
metadata:
  name: kafka-1
spec:
  ports:
  - name: plaintext
    port: 9092
    targetPort: 9092
    protocol: TCP
  - name: controller
    port: 9093
    targetPort: 9093
    protocol: TCP
  selector:
    app: kafka
    instance: kafka-1

---
apiVersion: v1
kind: Service
metadata:
  name: kafka-2
spec:
  ports:
  - name: plaintext
    port: 9092
    targetPort: 9092
    protocol: TCP
  - name: controller
    port: 9093
    targetPort: 9093
    protocol: TCP
  selector:
    app: kafka
    instance: kafka-2

---
apiVersion: v1
kind: Service
metadata:
  name: kafka-3
spec:
  ports:
  - name: plaintext
    port: 9092
    targetPort: 9092
    protocol: TCP
  - name: controller
    port: 9093
    targetPort: 9093
    protocol: TCP
  selector:
    app: kafka
    instance: kafka-3

NodePort 또는 LoadBalancer 타입으로 외부 접근 가능하게 하기?

접근 방법

  1. LoadBalancer로 서비스 생성

  2. 서비스가 생성된 후 kubectl get services 명령어를 사용하여 외부 IP 주소 확인. EXTERNAL-IP 열에 로드 밸런서의 IP가 표시될거임.

  3. 그 외부 아이피를 http://<외부 IP="">:9092 이런 형식으로 접근.

Deployment: 어플리케이션 배포 및 관리

Service: pod간의 통신, 외부 접근허용

Kube
kube
This post is licensed under CC BY 4.0 by the author.