From 56f5360618ca583834dc4a3b8eff43aaafa56a9d Mon Sep 17 00:00:00 2001 From: Jae-Joon Lee Date: Thu, 9 Jun 2022 10:08:36 +0900 Subject: [PATCH] add deployments files --- deploy/docker_hipsweb_sanic/Dockerfile | 44 +++ deploy/docker_hipsweb_sanic/Dockerfile_stiff | 12 + deploy/docker_hipsweb_sanic/README | 4 + .../deployment_hipsweb-sanic.yaml | 19 ++ deploy/docker_hipsweb_sanic/requirements.txt | 8 + deploy/docker_minio_cache_worker/Dockerfile | 10 + .../deployment_minio_cache_worker.yaml | 19 ++ .../requirements.txt | 4 + deploy/rabbitmq/README | 1 + deploy/rabbitmq/deployment_rabbitmq.yaml | 19 ++ deploy/rabbitmq/recommended.yaml | 306 ++++++++++++++++++ deploy/rabbitmq/service_rabbitmq.yaml | 9 + 12 files changed, 455 insertions(+) create mode 100644 deploy/docker_hipsweb_sanic/Dockerfile create mode 100644 deploy/docker_hipsweb_sanic/Dockerfile_stiff create mode 100644 deploy/docker_hipsweb_sanic/README create mode 100644 deploy/docker_hipsweb_sanic/deployment_hipsweb-sanic.yaml create mode 100644 deploy/docker_hipsweb_sanic/requirements.txt create mode 100644 deploy/docker_minio_cache_worker/Dockerfile create mode 100644 deploy/docker_minio_cache_worker/deployment_minio_cache_worker.yaml create mode 100644 deploy/docker_minio_cache_worker/requirements.txt create mode 100644 deploy/rabbitmq/README create mode 100644 deploy/rabbitmq/deployment_rabbitmq.yaml create mode 100644 deploy/rabbitmq/recommended.yaml create mode 100644 deploy/rabbitmq/service_rabbitmq.yaml diff --git a/deploy/docker_hipsweb_sanic/Dockerfile b/deploy/docker_hipsweb_sanic/Dockerfile new file mode 100644 index 0000000..9312176 --- /dev/null +++ b/deploy/docker_hipsweb_sanic/Dockerfile @@ -0,0 +1,44 @@ +FROM python:3.10 +#-alpine + +WORKDIR /usr/src/app + +# RUN apk add --no-cache --virtual .build-deps gcc make build-base tiff-dev +RUN apt-get install libtiff-dev + +COPY packages/stiff-2.7.1-master.tgz ./ +# RUN tar xvf stiff-2.7.1-master.tgz && cd stiff && autoreconf -f -i && ./configure && make +RUN tar xvf stiff-2.7.1-master.tgz && cd stiff && ./configure && make + + +FROM python:3.10 +#-alpine + +WORKDIR /usr/src/app + +# RUN apk add --no-cache --virtual .build-deps gcc make build-base imagemagick +RUN apt-get install -y imagemagick +# RUN apk add --no-cache --virtual tiff imagemagick +# RUN apk add --no-cache --virtual imagemagick + + +RUN pip install --no-cache-dir -U setuptools pip +RUN pip install --no-cache-dir uvloop + +COPY deploy/docker_hipsweb_sanic/requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +COPY packages/hips_cached/dist/hips_cached-0.11.dev0.tar.gz packages/shell_command_pool/dist/shell_command_pool-0.1.dev0.tar.gz ./ +RUN pip install hips_cached-0.11.dev0.tar.gz shell_command_pool-0.1.dev0.tar.gz + +COPY src/minio_cache_worker.py src/aio_run_stiff.py src/pull_and_jpg.py src/hipsweb_sanic.py src/minio_hosts.yaml ./ + +# COPY --from=0 /usr/src/app/stiff-2.4.0/src/stiff /usr/local/bin +COPY --from=0 /usr/src/app/stiff/src/stiff /usr/local/bin + +#RUN apk del .build-deps gcc make build-base + +#RUN pip install tqdm +#RUN apk add --no-cache imagemagick + +CMD [ "python", "hipsweb_sanic.py" ] diff --git a/deploy/docker_hipsweb_sanic/Dockerfile_stiff b/deploy/docker_hipsweb_sanic/Dockerfile_stiff new file mode 100644 index 0000000..e1c7e03 --- /dev/null +++ b/deploy/docker_hipsweb_sanic/Dockerfile_stiff @@ -0,0 +1,12 @@ +FROM python:3.10 +#-alpine + +WORKDIR /usr/src/app + +# RUN apk add --no-cache --virtual .build-deps gcc make build-base tiff-dev +RUN apt-get install libtiff-dev + +COPY packages/stiff-2.7.1-master.tgz ./ +# RUN tar xvf stiff-2.7.1-master.tgz && cd stiff && autoreconf -f -i && ./configure && make +RUN tar xvf stiff-2.7.1-master.tgz && cd stiff && ./configure && make + diff --git a/deploy/docker_hipsweb_sanic/README b/deploy/docker_hipsweb_sanic/README new file mode 100644 index 0000000..bc7b8b9 --- /dev/null +++ b/deploy/docker_hipsweb_sanic/README @@ -0,0 +1,4 @@ +# compiling stiff on alpine platform fails, likely due to incorrect use of global variables in siff source. +# My attempt to fix it, for now, failed. It compiles, but gives you a segfault. + +# As a workaround, use default (debian) distribution. diff --git a/deploy/docker_hipsweb_sanic/deployment_hipsweb-sanic.yaml b/deploy/docker_hipsweb_sanic/deployment_hipsweb-sanic.yaml new file mode 100644 index 0000000..77c9f75 --- /dev/null +++ b/deploy/docker_hipsweb_sanic/deployment_hipsweb-sanic.yaml @@ -0,0 +1,19 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hipsweb-sanic +spec: + replicas: 1 + selector: + matchLabels: + app: hipsweb-sanic + template: + metadata: + labels: + app: hipsweb-sanic + spec: + containers: + - name: hipsweb-sanic + image: registry.kasi.re.kr/hipsweb/hipsweb-sanic:latest + # command: ["celery"] + # args: ["HOSTNAME", "KUBERNETES_PORT"] diff --git a/deploy/docker_hipsweb_sanic/requirements.txt b/deploy/docker_hipsweb_sanic/requirements.txt new file mode 100644 index 0000000..2a2ee04 --- /dev/null +++ b/deploy/docker_hipsweb_sanic/requirements.txt @@ -0,0 +1,8 @@ +uvloop==0.16 +aiohttp +aiofiles +msgpack-python +celery +minio +sanic +pyyaml diff --git a/deploy/docker_minio_cache_worker/Dockerfile b/deploy/docker_minio_cache_worker/Dockerfile new file mode 100644 index 0000000..2d5fa58 --- /dev/null +++ b/deploy/docker_minio_cache_worker/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.7-alpine + +WORKDIR /usr/src/app + +COPY deploy/docker_minio_cache_worker/requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt + +COPY src/minio_cache_worker.py src/minio_hosts.yaml ./ + +CMD [ "celery", "-A", "minio_cache_worker", "worker", "-c", "2" ] diff --git a/deploy/docker_minio_cache_worker/deployment_minio_cache_worker.yaml b/deploy/docker_minio_cache_worker/deployment_minio_cache_worker.yaml new file mode 100644 index 0000000..06cf9f9 --- /dev/null +++ b/deploy/docker_minio_cache_worker/deployment_minio_cache_worker.yaml @@ -0,0 +1,19 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio-cache-worker +spec: + replicas: 1 + selector: + matchLabels: + app: minio-cache-worker + template: + metadata: + labels: + app: minio-cache-worker + spec: + containers: + - name: minio-cache-worker + image: registry.kasi.re.kr/hipsweb/minio_cache_worker:latest + # command: ["celery"] + # args: ["HOSTNAME", "KUBERNETES_PORT"] diff --git a/deploy/docker_minio_cache_worker/requirements.txt b/deploy/docker_minio_cache_worker/requirements.txt new file mode 100644 index 0000000..90ff100 --- /dev/null +++ b/deploy/docker_minio_cache_worker/requirements.txt @@ -0,0 +1,4 @@ +celery +minio +PyYAML>=5.1 +msgpack diff --git a/deploy/rabbitmq/README b/deploy/rabbitmq/README new file mode 100644 index 0000000..41bbf8a --- /dev/null +++ b/deploy/rabbitmq/README @@ -0,0 +1 @@ +# adopted from https://github.com/matiaslindgren/celery-kubernetes-example/tree/master/message_queue diff --git a/deploy/rabbitmq/deployment_rabbitmq.yaml b/deploy/rabbitmq/deployment_rabbitmq.yaml new file mode 100644 index 0000000..b3d64bb --- /dev/null +++ b/deploy/rabbitmq/deployment_rabbitmq.yaml @@ -0,0 +1,19 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rabbitmq +spec: + replicas: 1 + selector: + matchLabels: + app: rabbitmq + template: + metadata: + labels: + app: rabbitmq + spec: + containers: + - name: rabbitmq + image: rabbitmq:3-alpine + ports: + - containerPort: 5672 diff --git a/deploy/rabbitmq/recommended.yaml b/deploy/rabbitmq/recommended.yaml new file mode 100644 index 0000000..fb32819 --- /dev/null +++ b/deploy/rabbitmq/recommended.yaml @@ -0,0 +1,306 @@ +# Copyright 2017 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + name: kubernetes-dashboard + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kubernetes-dashboard + +--- + +kind: Service +apiVersion: v1 +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kubernetes-dashboard +spec: + ports: + - port: 443 + targetPort: 8443 + selector: + k8s-app: kubernetes-dashboard + +--- + +apiVersion: v1 +kind: Secret +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-certs + namespace: kubernetes-dashboard +type: Opaque + +--- + +apiVersion: v1 +kind: Secret +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-csrf + namespace: kubernetes-dashboard +type: Opaque +data: + csrf: "" + +--- + +apiVersion: v1 +kind: Secret +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-key-holder + namespace: kubernetes-dashboard +type: Opaque + +--- + +kind: ConfigMap +apiVersion: v1 +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-settings + namespace: kubernetes-dashboard + +--- + +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kubernetes-dashboard +rules: + # Allow Dashboard to get, update and delete Dashboard exclusive secrets. + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"] + verbs: ["get", "update", "delete"] + # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map. + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["kubernetes-dashboard-settings"] + verbs: ["get", "update"] + # Allow Dashboard to get metrics. + - apiGroups: [""] + resources: ["services"] + resourceNames: ["heapster", "dashboard-metrics-scraper"] + verbs: ["proxy"] + - apiGroups: [""] + resources: ["services/proxy"] + resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"] + verbs: ["get"] + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard +rules: + # Allow Metrics Scraper to get metrics from the Metrics server + - apiGroups: ["metrics.k8s.io"] + resources: ["pods", "nodes"] + verbs: ["get", "list", "watch"] + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kubernetes-dashboard +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kubernetes-dashboard +subjects: + - kind: ServiceAccount + name: kubernetes-dashboard + namespace: kubernetes-dashboard + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubernetes-dashboard +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubernetes-dashboard +subjects: + - kind: ServiceAccount + name: kubernetes-dashboard + namespace: kubernetes-dashboard + +--- + +kind: Deployment +apiVersion: apps/v1 +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kubernetes-dashboard +spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + k8s-app: kubernetes-dashboard + template: + metadata: + labels: + k8s-app: kubernetes-dashboard + spec: + securityContext: + seccompProfile: + type: RuntimeDefault + containers: + - name: kubernetes-dashboard + image: kubernetesui/dashboard:v2.5.0 + imagePullPolicy: Always + ports: + - containerPort: 8443 + protocol: TCP + args: + - --auto-generate-certificates + - --namespace=kubernetes-dashboard + # Uncomment the following line to manually specify Kubernetes API server Host + # If not specified, Dashboard will attempt to auto discover the API server and connect + # to it. Uncomment only if the default does not work. + # - --apiserver-host=http://my-address:port + volumeMounts: + - name: kubernetes-dashboard-certs + mountPath: /certs + # Create on-disk volume to store exec logs + - mountPath: /tmp + name: tmp-volume + livenessProbe: + httpGet: + scheme: HTTPS + path: / + port: 8443 + initialDelaySeconds: 30 + timeoutSeconds: 30 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsUser: 1001 + runAsGroup: 2001 + volumes: + - name: kubernetes-dashboard-certs + secret: + secretName: kubernetes-dashboard-certs + - name: tmp-volume + emptyDir: {} + serviceAccountName: kubernetes-dashboard + nodeSelector: + "kubernetes.io/os": linux + # Comment the following tolerations if Dashboard must not be deployed on master + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + +--- + +kind: Service +apiVersion: v1 +metadata: + labels: + k8s-app: dashboard-metrics-scraper + name: dashboard-metrics-scraper + namespace: kubernetes-dashboard +spec: + ports: + - port: 8000 + targetPort: 8000 + selector: + k8s-app: dashboard-metrics-scraper + +--- + +kind: Deployment +apiVersion: apps/v1 +metadata: + labels: + k8s-app: dashboard-metrics-scraper + name: dashboard-metrics-scraper + namespace: kubernetes-dashboard +spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + k8s-app: dashboard-metrics-scraper + template: + metadata: + labels: + k8s-app: dashboard-metrics-scraper + spec: + securityContext: + seccompProfile: + type: RuntimeDefault + containers: + - name: dashboard-metrics-scraper + image: kubernetesui/metrics-scraper:v1.0.7 + ports: + - containerPort: 8000 + protocol: TCP + livenessProbe: + httpGet: + scheme: HTTP + path: / + port: 8000 + initialDelaySeconds: 30 + timeoutSeconds: 30 + volumeMounts: + - mountPath: /tmp + name: tmp-volume + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsUser: 1001 + runAsGroup: 2001 + serviceAccountName: kubernetes-dashboard + nodeSelector: + "kubernetes.io/os": linux + # Comment the following tolerations if Dashboard must not be deployed on master + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + volumes: + - name: tmp-volume + emptyDir: {} diff --git a/deploy/rabbitmq/service_rabbitmq.yaml b/deploy/rabbitmq/service_rabbitmq.yaml new file mode 100644 index 0000000..fd13a13 --- /dev/null +++ b/deploy/rabbitmq/service_rabbitmq.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Service +metadata: + name: rabbitmq +spec: + selector: + app: rabbitmq + ports: + - port: 5672 -- GitLab