diff --git a/flink/README.md b/flink/README.md index afd73ae..cf30748 100644 --- a/flink/README.md +++ b/flink/README.md @@ -55,9 +55,36 @@ image](https://hub.docker.com/_/flink). docker run -d clearlinux/flink ``` + ### Deploy with Kubernetes +This image can also be deployed on a Kubernetes cluster, such as [minikube](https://kubernetes.io/docs/setup/learning-environment/minikube/).The following example YAML files are provided in the repository as reference for Kubernetes deployment: + +- [`flink-deployment.yaml`](https://github.com/clearlinux/dockerfiles/blob/master/flink/flink-deployment.yaml): example using default configuration to create a basic flink service. + +- [`flink-deployment-conf.yaml`](https://github.com/clearlinux/dockerfiles/blob/master/flink/flink-deployment-conf.yaml): example using your own custom configuration to create a flink service. + + + +Steps to deploy flink on a Kubernetes cluster: + +1. If you want to deploy `flink-deployment.yaml` + + ``` + kubectl create -f flink-deployment.yaml + ``` + + Or if you want to deploy `flink-deployment-conf.yaml` + + ``` + kubectl create -f flink-deployment-conf.yaml + ``` + +2. Run kubectl proxy in your terminal. + +3. Navigate to [http://\:8001/api/v1/namespaces/default/services/flink-jobmanager:ui/proxy](http://\:8001/api/v1/namespaces/default/services/flink-jobmanager:ui/proxy) in your browser. + ## Build and modify: diff --git a/flink/flink-deployment-conf.yaml b/flink/flink-deployment-conf.yaml new file mode 100644 index 0000000..7737e84 --- /dev/null +++ b/flink/flink-deployment-conf.yaml @@ -0,0 +1,134 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: flink-config + labels: + app: flink +data: + flink-conf.yaml: |+ + jobmanager.rpc.address: flink-jobmanager + taskmanager.numberOfTaskSlots: 1 + blob.server.port: 6124 + jobmanager.rpc.port: 6123 + taskmanager.rpc.port: 6122 + jobmanager.heap.size: 1024m + taskmanager.heap.size: 1000m + log4j.properties: |+ + log4j.rootLogger=INFO, file + log4j.logger.akka=INFO + log4j.logger.org.apache.kafka=INFO + log4j.logger.org.apache.hadoop=INFO + log4j.logger.org.apache.zookeeper=INFO + log4j.appender.file=org.apache.log4j.FileAppender + log4j.appender.file.file=${log.file} + log4j.appender.file.layout=org.apache.log4j.PatternLayout + log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n + log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file + +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: flink-jobmanager +spec: + replicas: 1 + template: + metadata: + labels: + app: flink + component: jobmanager + spec: + containers: + - name: jobmanager + image: docker.io/clearlinux/flink + args: + - jobmanager + ports: + - containerPort: 6123 + name: rpc + - containerPort: 6124 + name: blob + - containerPort: 6125 + name: query + - containerPort: 8081 + name: ui + livenessProbe: + tcpSocket: + port: 6122 + initialDelaySeconds: 30 + periodSeconds: 60 + volumeMounts: + - name: flink-config-volume + mountPath: /usr/share/defaults/flink + volumes: + - name: flink-config-volume + configMap: + name: flink-config + items: + - key: flink-conf.yaml + path: flink-conf.yaml + - key: log4j.properties + path: log4j.properties + +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: flink-taskmanager +spec: + replicas: 2 + template: + metadata: + labels: + app: flink + component: taskmanager + spec: + containers: + - name: taskmanager + image: docker.io/clearlinux/flink + args: + - taskmanager + ports: + - containerPort: 6121 + name: data + - containerPort: 6122 + name: rpc + - containerPort: 6125 + name: query + livenessProbe: + tcpSocket: + port: 6122 + initialDelaySeconds: 30 + periodSeconds: 60 + volumeMounts: + - name: flink-config-volume + mountPath: /usr/share/defaults/flink + volumes: + - name: flink-config-volume + configMap: + name: flink-config + items: + - key: flink-conf.yaml + path: flink-conf.yaml + - key: log4j.properties + path: log4j.properties + +--- +apiVersion: v1 +kind: Service +metadata: + name: flink-jobmanager +spec: + ports: + - name: rpc + port: 6123 + - name: blob + port: 6124 + - name: query + port: 6125 + - name: ui + port: 8081 + selector: + app: flink + component: jobmanager + diff --git a/flink/flink-deployment.yaml b/flink/flink-deployment.yaml new file mode 100644 index 0000000..2223cc5 --- /dev/null +++ b/flink/flink-deployment.yaml @@ -0,0 +1,77 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: flink-jobmanager +spec: + replicas: 1 + template: + metadata: + labels: + app: flink + component: jobmanager + spec: + containers: + - name: jobmanager + image: docker.io/clearlinux/flink + args: + - jobmanager + ports: + - containerPort: 6123 + name: rpc + - containerPort: 6124 + name: blob + - containerPort: 6125 + name: query + - containerPort: 8081 + name: ui + env: + - name: JOB_MANAGER_RPC_ADDRESS + value: flink-jobmanager + +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: flink-taskmanager +spec: + replicas: 2 + template: + metadata: + labels: + app: flink + component: taskmanager + spec: + containers: + - name: taskmanager + image: docker.io/clearlinux/flink + args: + - taskmanager + ports: + - containerPort: 6121 + name: data + - containerPort: 6122 + name: rpc + - containerPort: 6125 + name: query + env: + - name: JOB_MANAGER_RPC_ADDRESS + value: flink-jobmanager + +--- +apiVersion: v1 +kind: Service +metadata: + name: flink-jobmanager +spec: + ports: + - name: rpc + port: 6123 + - name: blob + port: 6124 + - name: query + port: 6125 + - name: ui + port: 8081 + selector: + app: flink + component: jobmanager