From bd6ac83627cf7dfc56c40dc298a2085cc3c6fef9 Mon Sep 17 00:00:00 2001
From: Johann DUTON-MORGAND
<datalake-duton-morgand.consultant@dgfip.finances.gouv.fr>
Date: Tue, 9 Jul 2024 02:01:26 +0200
Subject: [PATCH] [make] new recipes added (helm charts deployments)
---
Makefile | 135 +++++++++++++++++++--
README.md | 23 +++-
k8s/charts/README.md | 54 +++++++--
k8s/charts/_external-components_/README.md | 116 ++----------------
4 files changed, 189 insertions(+), 139 deletions(-)
diff --git a/Makefile b/Makefile
index e98dfd0..06ab1a4 100644
--- a/Makefile
+++ b/Makefile
@@ -1,11 +1,11 @@
-.PHONY: api web proxy-args base launch-docker-compose help
+.PHONY: api web base proxy-args charts-secrets deploy-ext-charts deploy-app-charts deploy-qdrant deploy-redis deploy-minio deploy-mongodb deploy-postgresql deploy-mlflow deploy-caradoc-api deploy-caradoc-web docker-compose help
SHELL := /bin/bash
-EMBEDDING_DIM ?= 1024
-
-use_ollama = 0
use_proxy = 0
+use_ollama = 0
+openai_api_key = xxxxxxxx
+EMBEDDING_DIM ?= 1024
LLM_OLLAMA ?=mistral
EMBEDDING_OLLAMA ?=mxbai-embed-large
@@ -13,6 +13,9 @@ base_api_img_tag = base-r1
api_img_tag = 1.0
web_img_tag = 1.0
+k8s_namespace = caradoc
+k8s_target_env = dev
+
HTTP_PROXY ?= $(http_proxy)
HTTPS_PROXY ?= $(https_proxy)
@@ -24,9 +27,22 @@ MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
MKFILE_DIR := $(dir ${MKFILE_PATH})
+define __DEPLOY_CHART__
+ @echo "" && \
+ echo "deploying $(1) ..." && \
+ cd $(2) && \
+ helm upgrade --install $(1) $(1)/ --reset-values -f $(strip $(k8s_target_env))-$(strip $(1)).values.yaml --namespace $(k8s_namespace)
+endef
+
+define DEPLOY_CHART
+ $(call __DEPLOY_CHART__, $(1), "k8s/charts/_external-components_")
+endef
+
+
api: ## build API docker image
@cd api && docker build --no-cache -t ${API_IMG_NAME} .
+
web: ## build Web docker image
@echo "use_proxy: $(use_proxy)";
ifeq ($(use_proxy), 1)
@@ -41,14 +57,6 @@ else
-t ${WEB_IMG_NAME} .
endif
-proxy-args: ## print used proxy env vars if needed (@see use_proxy)
- @echo "use_proxy: $(use_proxy)"
-ifeq ($(use_proxy), 1)
- @echo "HTTP_PROXY: $(HTTP_PROXY)" && \
- echo "HTTPS_PROXY: $(HTTPS_PROXY)";
-else
- @echo "no proxy will be used to build the docker images.";
-endif
base: ## build API base docker image
@echo "use_proxy: $(use_proxy)";
@@ -64,7 +72,108 @@ else
-t ${BASE_API_IMG_NAME} .
endif
-launch-docker-compose: ## launch application with docker containers
+
+proxy-args: ## print used proxy env vars if needed (@see use_proxy)
+ @echo "use_proxy: $(use_proxy)"
+ifeq ($(use_proxy), 1)
+ @echo "HTTP_PROXY: $(HTTP_PROXY)" && \
+ echo "HTTPS_PROXY: $(HTTPS_PROXY)";
+else
+ @echo "no proxy will be used to build the docker images.";
+endif
+
+
+charts-secrets: ## create helm charts secrets
+charts-secrets: MONGODB_ROOT_USER=root
+charts-secrets: MONGODB_ROOT_PASSWORD::=$(shell uuidgen -r)
+charts-secrets:
+ifeq ($(openai_api_key), xxxxxxxx)
+ @echo "[ WARNING ] openai_api_key is not set ! => 'make openai_api_key=... charts-secrets'" && \
+ echo ""
+endif
+ @kubectl create secret generic redis-secret \
+ --namespace $(k8s_namespace) \
+ --from-literal db-password=$(shell uuidgen -r) \
+ --dry-run=client -o yaml > $(strip $(MKFILE_DIR))redis-secret.yaml \
+ && kubectl apply -f $(strip $(MKFILE_DIR))redis-secret.yaml && \
+ \
+ kubectl create secret generic minio-secret \
+ --namespace $(k8s_namespace) \
+ --from-literal minio-root-user=admin \
+ --from-literal minio-root-password=$(shell uuidgen -r) \
+ --dry-run=client -o yaml > $(strip $(MKFILE_DIR))minio-secret.yaml \
+ && kubectl apply -f $(strip $(MKFILE_DIR))minio-secret.yaml && \
+ \
+ kubectl create secret generic mongodb-secret \
+ --namespace $(k8s_namespace) \
+ --from-literal mongodb-root-user=$(MONGODB_ROOT_USER) \
+ --from-literal mongodb-root-password=$(MONGODB_ROOT_PASSWORD) \
+ --from-literal mongodb-metrics-password=$(shell uuidgen -r) \
+ --from-literal mongodb-uri="mongodb://$(strip $(MONGODB_ROOT_USER)):$(strip $(MONGODB_ROOT_PASSWORD))@mongodb:27017/test?serverSelectionTimeoutMS=3000&authSource=admin" \
+ --dry-run=client -o yaml > $(strip $(MKFILE_DIR))mongodb-secret.yaml \
+ && kubectl apply -f $(strip $(MKFILE_DIR))mongodb-secret.yaml && \
+ \
+ kubectl create secret generic postgresql-secret \
+ --namespace $(k8s_namespace) \
+ --from-literal postgres-password=$(shell uuidgen -r) \
+ --from-literal password=$(shell uuidgen -r) \
+ --dry-run=client -o yaml > $(strip $(MKFILE_DIR))postgresql-secret.yaml \
+ && kubectl apply -f $(strip $(MKFILE_DIR))postgresql-secret.yaml && \
+ \
+ kubectl create secret generic mlflow-secret \
+ --namespace $(k8s_namespace) \
+ --from-literal mlflow-user=admin \
+ --from-literal mlflow-password=$(shell uuidgen -r) \
+ --dry-run=client -o yaml > $(strip $(MKFILE_DIR))mlflow-secret.yaml \
+ && kubectl apply -f $(strip $(MKFILE_DIR))mlflow-secret.yaml && \
+ \
+ kubectl create secret generic caradoc-api-secret \
+ --namespace $(k8s_namespace) \
+ --from-literal openai-api-key=$(strip $(openai_api_key)) \
+ --dry-run=client -o yaml > $(strip $(MKFILE_DIR))caradoc-api-secret.yaml \
+ && kubectl apply -f $(strip $(MKFILE_DIR))caradoc-api-secret.yaml
+
+
+deploy-ext-charts: deploy-qdrant deploy-redis deploy-minio deploy-mongodb deploy-postgresql deploy-mlflow ## deploy all external helm charts
+
+deploy-app-charts: deploy-caradoc-api deploy-caradoc-web ## deploy application helm charts (API & Web)
+
+deploy-qdrant: ## deploy qdrant helm chart
+ $(call DEPLOY_CHART, qdrant)
+
+deploy-redis: ## deploy redis helm chart
+ $(call DEPLOY_CHART, redis)
+
+deploy-minio: ## deploy minio helm chart
+ $(call DEPLOY_CHART, minio)
+
+deploy-mongodb: ## deploy mongodb helm chart
+ $(call DEPLOY_CHART, mongodb)
+
+deploy-postgresql: POSTGRES_PASSWORD=$(shell kubectl get secret --namespace $(k8s_namespace) postgresql-secret -o jsonpath="{.data.postgres-password}" | base64 --decode) ## deploy postgresql helm chart (+ mlflow database creation)
+deploy-postgresql:
+ $(call DEPLOY_CHART, postgresql)
+ @echo "" && \
+ echo "Creating the required database for the mlflow pod ..." && \
+ echo "" && \
+ sleep 45 && \
+ kubectl run postgresql-client --rm --tty -i --restart='Never' \
+ --namespace $(k8s_namespace) \
+ --image docker.io/bitnami/postgresql:14.3.0-debian-10-r20 \
+ --env="PGPASSWORD=$(strip $(POSTGRES_PASSWORD))" \
+ --command -- psql -h postgresql -U postgres -d postgres -p 5432 -c "CREATE DATABASE mlflow OWNER postgres;"
+
+deploy-mlflow: ## deploy mlflow helm chart
+ $(call DEPLOY_CHART, mlflow)
+
+deploy-caradoc-api: ## deploy caradoc API helm chart
+ $(call __DEPLOY_CHART__, caradoc-api, "k8s/charts")
+
+deploy-caradoc-web: ## deploy caradoc Web helm chart
+ $(call __DEPLOY_CHART__, caradoc-web, "k8s/charts")
+
+
+docker-compose: ## launch application with docker-compose
@echo "use_proxy: $(use_proxy)";
ifeq ($(use_proxy), 1)
@cd docker_compose && \
diff --git a/README.md b/README.md
index 4af33e1..b4d5d02 100644
--- a/README.md
+++ b/README.md
@@ -75,11 +75,23 @@ Le Makefile du dépôt permet de construire les images Docker:
```bash
$> make help
+
api build API docker image
web build Web docker image
-proxy-args print used proxy env vars if needed (@see use_proxy)
base build API base docker image
-launch-docker-compose launch application with docker containers
+proxy-args print used proxy env vars if needed (@see use_proxy)
+charts-secrets create helm charts secrets
+deploy-ext-charts deploy all external helm charts
+deploy-app-charts deploy application helm charts (API & Web)
+deploy-qdrant deploy qdrant helm chart
+deploy-redis deploy redis helm chart
+deploy-minio deploy minio helm chart
+deploy-mongodb deploy mongodb helm chart
+deploy-postgresql deploy postgresql helm chart (+ mlflow database creation)
+deploy-mlflow deploy mlflow helm chart
+deploy-caradoc-api deploy caradoc API helm chart
+deploy-caradoc-web deploy caradoc Web helm chart
+docker-compose launch application with docker-compose
help print this help
```
@@ -125,17 +137,18 @@ Ensuite, il sera possible de lancer le Minio et le MLFlow.
Par la suite, pour lancer l'application, il suffit de lancer le script suivant :
```bash
-make launch-docker-compose EMBEDDING_DIM=1024
-# make launch-docker-compose use_ollama=1 LLM_OLLAMA=mistral EMBEDDING_OLLAMA=mxbai-embed-large EMBEDDING_DIM=1024
+make docker-compose EMBEDDING_DIM=1024
+# make docker-compose use_ollama=1 LLM_OLLAMA=mistral EMBEDDING_OLLAMA=mxbai-embed-large EMBEDDING_DIM=1024
```
Cela lancera notre application ainsi que le script permettant de créer une collection dans qdrant au nom : `caradoc`
-NB : Il est possible de lancer l'application avec un serveru ollama directement avec le flag use_ollama=1
+NB : Il est possible de lancer l'application avec un serveur ollama directement avec le flag use_ollama=1
Le choix du modèle est disponible à l'aide des deux arguments : LLM_OLLAMA et EMBEDDING_OLLAMA
Les modèles disponibles sont dans [la documentation d'Ollama](https://ollama.com/library)
+
### 3. Installation via Helm
Charts Helm [ici](./k8s/charts/README.md)
diff --git a/k8s/charts/README.md b/k8s/charts/README.md
index 4615db6..5414776 100644
--- a/k8s/charts/README.md
+++ b/k8s/charts/README.md
@@ -1,35 +1,63 @@
-## Prérequis
-Installer les charts des composants externes [cf.](./\_external-components\_/README.md)
+## Pré-requis
+ - Un cluster Kubernetes (K8s);
+ - kubectl (https://kubernetes.io/fr/docs/tasks/tools/install-kubectl/);
+ - Helm (https://github.com/helm/helm);
+ - NFS provisioner (https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner).
-## caradoc-api
-Renseigner la valeur de l'OPENAI API KEY avant de créer la ressource k8s secret associée
+## Recettes du Makefile relatives aux charts Helm
```sh
-kubectl create secret generic caradoc-api-secret \
- --namespace caradoc \
- --from-literal openai-api-key=xxxxxxxx \
---dry-run=client -o yaml > caradoc-api-secret.yaml
+$> make help | grep -i chart
+
+charts-secrets create helm charts secrets
+deploy-ext-charts deploy all external helm charts
+deploy-app-charts deploy application helm charts (API & Web)
+deploy-qdrant deploy qdrant helm chart
+deploy-redis deploy redis helm chart
+deploy-minio deploy minio helm chart
+deploy-mongodb deploy mongodb helm chart
+deploy-postgresql deploy postgresql helm chart (+ mlflow database creation)
+deploy-mlflow deploy mlflow helm chart
+deploy-caradoc-api deploy caradoc API helm chart
+deploy-caradoc-web deploy caradoc Web helm chart
```
+
+## Déploiement standard
+
+Renseigner les valeurs par défaut pour l'environnement cible dans le fichier Makefile (namespace, ...).
+
+
+### Créer les secrets K8s
+
+```sh
+
+$> make openai_api_key=<à définir> charts-secrets
+
+```
+
+### Déployer les charts des composants dont dépend Caradoc
+
+Configurer/ajuster les valeurs des fichiers \*.values.yaml pour l'environnement cible avant les déploiements.
+
```sh
-helm upgrade --install caradoc-api caradoc-api/ --reset-values -f dev-caradoc-api.values.yaml --namespace caradoc
+$> make deploy-ext-charts
```
-## caradoc-web
+### Déployer les charts applicatifs (caradoc-api & caradoc-web)
-Renseigner la valeur de l'URI public de MLflow dans le fichier \*.values.yaml avant le déploiement
+Configurer/ajuster les valeurs (notamment les différents endpoints) des fichiers \*.values.yaml pour l'environnement cible avant les déploiements.
```sh
-helm upgrade --install caradoc-web caradoc-web/ --reset-values -f dev-caradoc-web.values.yaml --namespace caradoc
+$> make deploy-app-charts
```
-NB : Il ne faut pas oublier de renseigner les différents endpoints dans les fichiers values de l'API et du Web
diff --git a/k8s/charts/_external-components_/README.md b/k8s/charts/_external-components_/README.md
index c483e22..ee38c44 100644
--- a/k8s/charts/_external-components_/README.md
+++ b/k8s/charts/_external-components_/README.md
@@ -1,127 +1,27 @@
-## Prérequis
- - NFS provisioner pour la persistence
+## Références
-## qdrant
+### qdrant
https://artifacthub.io/packages/helm/qdrant/qdrant
-```sh
-helm upgrade --install qdrant qdrant/ --reset-values -f dev-qdrant.values.yaml --namespace caradoc
-
-```
-
-
-## redis (bitnami)
+### redis (bitnami)
https://artifacthub.io/packages/helm/bitnami/redis
-```sh
-
-kubectl create secret generic redis-secret \
- --namespace caradoc \
- --from-literal db-password=$(uuidgen -r) \
- --dry-run=client -o yaml > redis-secret.yaml
-
-```
-```sh
-
-helm upgrade --install redis redis/ --reset-values -f dev-redis.values.yaml --namespace caradoc
-
-```
-
-
-## minio (bitnami)
+### minio (bitnami)
https://artifacthub.io/packages/helm/bitnami/minio
-```sh
-
-kubectl create secret generic minio-secret \
- --namespace caradoc \
- --from-literal minio-root-user=admin \
- --from-literal minio-root-password=$(uuidgen -r) \
- --dry-run=client -o yaml > minio-secret.yaml
-```
-
-```sh
-
-helm upgrade --install minio minio/ --reset-values -f dev-minio.values.yaml --namespace caradoc
-
-```
-
-
-## mongodb (bitnami)
+### mongodb (bitnami)
https://artifacthub.io/packages/helm/bitnami/mongodb
-https://github.com/bitnami/containers/blob/main/bitnami/mongodb/README.md
-
-```sh
-
-export MONGODB_ROOT_USER=root && \
-export MONGODB_ROOT_PASSWORD=$(uuidgen -r) && \
-kubectl create secret generic mongodb-secret \
- --namespace caradoc \
- --from-literal mongodb-root-user=$MONGODB_ROOT_USER \
- --from-literal mongodb-root-password=$MONGODB_ROOT_PASSWORD \
- --from-literal mongodb-metrics-password=$(uuidgen -r) \
- --from-literal mongodb-uri="mongodb://$MONGODB_ROOT_USER:$MONGODB_ROOT_PASSWORD@mongodb:27017/test?serverSelectionTimeoutMS=3000&authSource=admin" \
- --dry-run=client -o yaml > mongodb-secret.yaml
-
-```
-```sh
-
-helm upgrade --install mongodb mongodb/ --reset-values -f dev-mongodb.values.yaml --namespace caradoc
-
-```
+https://github.com/bitnami/containers/blob/main/bitnami/mongodb/README.md
-## postgresql (bitnami)
+### postgresql (bitnami)
https://artifacthub.io/packages/helm/bitnami/postgresql
-```sh
-
-kubectl create secret generic postgresql-secret \
- --namespace caradoc \
- --from-literal postgres-password=$(uuidgen -r) \
- --from-literal password=$(uuidgen -r) \
- --dry-run=client -o yaml > postgresql-secret.yaml
-
-```
-
-```sh
-
-helm upgrade --install postgresql postgresql/ --reset-values -f dev-postgresql.values.yaml --namespace caradoc
-
-```
-
-### Créer la base de données mlflow
-```sh
-
-export POSTGRES_PASSWORD=$(kubectl get secret --namespace caradoc postgresql-secret -o jsonpath="{.data.postgres-password}" | base64 --decode)
-
-kubectl run postgresql-client --rm --tty -i --restart='Never' \
- --namespace caradoc \
- --image docker.io/bitnami/postgresql:14.3.0-debian-10-r20 \
- --env="PGPASSWORD=$POSTGRES_PASSWORD" \
- --command -- psql --host postgresql -U postgres -d postgres -p 5432
-
-CREATE DATABASE mlflow OWNER postgres;
-
-```
-
-## mlflow (bitnami)
+### mlflow (bitnami)
https://artifacthub.io/packages/helm/bitnami/mlflow
-```sh
-
-kubectl create secret generic mlflow-secret \
- --namespace caradoc \
- --from-literal mlflow-user=admin \
- --from-literal mlflow-password=$(uuidgen -r) \
- --dry-run=client -o yaml > mlflow-secret.yaml
-
-helm upgrade --install mlflow mlflow/ --reset-values -f dev-mlflow.values.yaml --namespace caradoc
-
-```
-
--
GitLab