Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • dgfip/projets-ia/caradoc
  • blenzi/caradoc
2 results
Show changes
......@@ -3,8 +3,12 @@
api/tmp
.idea
*-secret.yaml
int-*.values.yaml
/*-secret.yaml
/k8s/charts/int-*.values.yaml
/k8s/charts/test-*.values.yaml
/k8s/charts/_external-components_/int-*.values.yaml
/k8s/charts/_external-components_/test-*.values.yaml
/k8s/*.rbac.yaml
/docker_compose/secret*
/docker_compose/external_components/minio_storage
......
.PHONY: api web proxy-args base launch-docker-compose help
.PHONY: api web base proxy-args charts-secrets deploy-ext-charts deploy-app-charts deploy-qdrant deploy-redis deploy-minio deploy-mongodb deploy-postgresql deploy-mlflow deploy-caradoc-api deploy-caradoc-web docker-compose help
SHELL := /bin/bash
EMBEDDING_DIM ?= 1024
use_ollama = 0
use_proxy = 0
use_ollama = 0
openai_api_key = xxxxxxxx
EMBEDDING_DIM ?= 1024
LLM_OLLAMA ?=mistral
EMBEDDING_OLLAMA ?=mxbai-embed-large
......@@ -13,6 +13,9 @@ base_api_img_tag = base-r1
api_img_tag = 1.0
web_img_tag = 1.0
k8s_namespace = caradoc
k8s_target_env = dev
HTTP_PROXY ?= $(http_proxy)
HTTPS_PROXY ?= $(https_proxy)
......@@ -24,9 +27,22 @@ MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
MKFILE_DIR := $(dir ${MKFILE_PATH})
define __DEPLOY_CHART__
@echo "" && \
echo "deploying $(1) ..." && \
cd $(2) && \
helm upgrade --install $(1) $(1)/ --reset-values -f $(strip $(k8s_target_env))-$(strip $(1)).values.yaml --namespace $(k8s_namespace)
endef
define DEPLOY_CHART
$(call __DEPLOY_CHART__, $(1), "k8s/charts/_external-components_")
endef
api: ## build API docker image
@cd api && docker build --no-cache -t ${API_IMG_NAME} .
web: ## build Web docker image
@echo "use_proxy: $(use_proxy)";
ifeq ($(use_proxy), 1)
......@@ -41,14 +57,6 @@ else
-t ${WEB_IMG_NAME} .
endif
proxy-args: ## print used proxy env vars if needed (@see use_proxy)
@echo "use_proxy: $(use_proxy)"
ifeq ($(use_proxy), 1)
@echo "HTTP_PROXY: $(HTTP_PROXY)" && \
echo "HTTPS_PROXY: $(HTTPS_PROXY)";
else
@echo "no proxy will be used to build the docker images.";
endif
base: ## build API base docker image
@echo "use_proxy: $(use_proxy)";
......@@ -64,7 +72,108 @@ else
-t ${BASE_API_IMG_NAME} .
endif
launch-docker-compose: ## launch application with docker containers
proxy-args: ## print used proxy env vars if needed (@see use_proxy)
@echo "use_proxy: $(use_proxy)"
ifeq ($(use_proxy), 1)
@echo "HTTP_PROXY: $(HTTP_PROXY)" && \
echo "HTTPS_PROXY: $(HTTPS_PROXY)";
else
@echo "no proxy will be used to build the docker images.";
endif
charts-secrets: ## create helm charts secrets
charts-secrets: MONGODB_ROOT_USER=root
charts-secrets: MONGODB_ROOT_PASSWORD::=$(shell uuidgen -r)
charts-secrets:
ifeq ($(openai_api_key), xxxxxxxx)
@echo "[ WARNING ] openai_api_key is not set ! => 'make openai_api_key=... charts-secrets'" && \
echo ""
endif
@kubectl create secret generic redis-secret \
--namespace $(k8s_namespace) \
--from-literal db-password=$(shell uuidgen -r) \
--dry-run=client -o yaml > $(strip $(MKFILE_DIR))redis-secret.yaml \
&& kubectl apply -f $(strip $(MKFILE_DIR))redis-secret.yaml && \
\
kubectl create secret generic minio-secret \
--namespace $(k8s_namespace) \
--from-literal minio-root-user=admin \
--from-literal minio-root-password=$(shell uuidgen -r) \
--dry-run=client -o yaml > $(strip $(MKFILE_DIR))minio-secret.yaml \
&& kubectl apply -f $(strip $(MKFILE_DIR))minio-secret.yaml && \
\
kubectl create secret generic mongodb-secret \
--namespace $(k8s_namespace) \
--from-literal mongodb-root-user=$(MONGODB_ROOT_USER) \
--from-literal mongodb-root-password=$(MONGODB_ROOT_PASSWORD) \
--from-literal mongodb-metrics-password=$(shell uuidgen -r) \
--from-literal mongodb-uri="mongodb://$(strip $(MONGODB_ROOT_USER)):$(strip $(MONGODB_ROOT_PASSWORD))@mongodb:27017/test?serverSelectionTimeoutMS=3000&authSource=admin" \
--dry-run=client -o yaml > $(strip $(MKFILE_DIR))mongodb-secret.yaml \
&& kubectl apply -f $(strip $(MKFILE_DIR))mongodb-secret.yaml && \
\
kubectl create secret generic postgresql-secret \
--namespace $(k8s_namespace) \
--from-literal postgres-password=$(shell uuidgen -r) \
--from-literal password=$(shell uuidgen -r) \
--dry-run=client -o yaml > $(strip $(MKFILE_DIR))postgresql-secret.yaml \
&& kubectl apply -f $(strip $(MKFILE_DIR))postgresql-secret.yaml && \
\
kubectl create secret generic mlflow-secret \
--namespace $(k8s_namespace) \
--from-literal mlflow-user=admin \
--from-literal mlflow-password=$(shell uuidgen -r) \
--dry-run=client -o yaml > $(strip $(MKFILE_DIR))mlflow-secret.yaml \
&& kubectl apply -f $(strip $(MKFILE_DIR))mlflow-secret.yaml && \
\
kubectl create secret generic caradoc-api-secret \
--namespace $(k8s_namespace) \
--from-literal openai-api-key=$(strip $(openai_api_key)) \
--dry-run=client -o yaml > $(strip $(MKFILE_DIR))caradoc-api-secret.yaml \
&& kubectl apply -f $(strip $(MKFILE_DIR))caradoc-api-secret.yaml
deploy-ext-charts: deploy-qdrant deploy-redis deploy-minio deploy-mongodb deploy-postgresql deploy-mlflow ## deploy all external helm charts
deploy-app-charts: deploy-caradoc-api deploy-caradoc-web ## deploy application helm charts (API & Web)
deploy-qdrant: ## deploy qdrant helm chart
$(call DEPLOY_CHART, qdrant)
deploy-redis: ## deploy redis helm chart
$(call DEPLOY_CHART, redis)
deploy-minio: ## deploy minio helm chart
$(call DEPLOY_CHART, minio)
deploy-mongodb: ## deploy mongodb helm chart
$(call DEPLOY_CHART, mongodb)
deploy-postgresql: POSTGRES_PASSWORD=$(shell kubectl get secret --namespace $(k8s_namespace) postgresql-secret -o jsonpath="{.data.postgres-password}" | base64 --decode) ## deploy postgresql helm chart (+ mlflow database creation)
deploy-postgresql:
$(call DEPLOY_CHART, postgresql)
@echo "" && \
echo "Creating the required database for the mlflow pod ..." && \
echo "" && \
sleep 45 && \
kubectl run postgresql-client --rm --tty -i --restart='Never' \
--namespace $(k8s_namespace) \
--image docker.io/bitnami/postgresql:14.3.0-debian-10-r20 \
--env="PGPASSWORD=$(strip $(POSTGRES_PASSWORD))" \
--command -- psql -h postgresql -U postgres -d postgres -p 5432 -c "CREATE DATABASE mlflow OWNER postgres;"
deploy-mlflow: ## deploy mlflow helm chart
$(call DEPLOY_CHART, mlflow)
deploy-caradoc-api: ## deploy caradoc API helm chart
$(call __DEPLOY_CHART__, caradoc-api, "k8s/charts")
deploy-caradoc-web: ## deploy caradoc Web helm chart
$(call __DEPLOY_CHART__, caradoc-web, "k8s/charts")
docker-compose: ## launch application with docker-compose
@echo "use_proxy: $(use_proxy)";
ifeq ($(use_proxy), 1)
@cd docker_compose && \
......
......@@ -75,11 +75,23 @@ Le Makefile du dépôt permet de construire les images Docker:
```bash
$> make help
api build API docker image
web build Web docker image
proxy-args print used proxy env vars if needed (@see use_proxy)
base build API base docker image
launch-docker-compose launch application with docker containers
proxy-args print used proxy env vars if needed (@see use_proxy)
charts-secrets create helm charts secrets
deploy-ext-charts deploy all external helm charts
deploy-app-charts deploy application helm charts (API & Web)
deploy-qdrant deploy qdrant helm chart
deploy-redis deploy redis helm chart
deploy-minio deploy minio helm chart
deploy-mongodb deploy mongodb helm chart
deploy-postgresql deploy postgresql helm chart (+ mlflow database creation)
deploy-mlflow deploy mlflow helm chart
deploy-caradoc-api deploy caradoc API helm chart
deploy-caradoc-web deploy caradoc Web helm chart
docker-compose launch application with docker-compose
help print this help
```
......@@ -125,17 +137,18 @@ Ensuite, il sera possible de lancer le Minio et le MLFlow.
Par la suite, pour lancer l'application, il suffit de lancer le script suivant :
```bash
make launch-docker-compose EMBEDDING_DIM=1024
# make launch-docker-compose use_ollama=1 LLM_OLLAMA=mistral EMBEDDING_OLLAMA=mxbai-embed-large EMBEDDING_DIM=1024
make docker-compose EMBEDDING_DIM=1024
# make docker-compose use_ollama=1 LLM_OLLAMA=mistral EMBEDDING_OLLAMA=mxbai-embed-large EMBEDDING_DIM=1024
```
Cela lancera notre application ainsi que le script permettant de créer une collection dans qdrant au nom : `caradoc`
NB : Il est possible de lancer l'application avec un serveru ollama directement avec le flag use_ollama=1
NB : Il est possible de lancer l'application avec un serveur ollama directement avec le flag use_ollama=1
Le choix du modèle est disponible à l'aide des deux arguments : LLM_OLLAMA et EMBEDDING_OLLAMA
Les modèles disponibles sont dans [la documentation d'Ollama](https://ollama.com/library)
### 3. Installation via Helm
Charts Helm [ici](./k8s/charts/README.md)
......
## Prérequis
Installer les charts des composants externes [cf.](./\_external-components\_/README.md)
## Pré-requis
- Un cluster Kubernetes (K8s);
- kubectl (https://kubernetes.io/fr/docs/tasks/tools/install-kubectl/);
- Helm (https://github.com/helm/helm);
- NFS provisioner (https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner).
## caradoc-api
Renseigner la valeur de l'OPENAI API KEY avant de créer la ressource k8s secret associée
## Recettes du Makefile relatives aux charts Helm
```sh
kubectl create secret generic caradoc-api-secret \
--namespace caradoc \
--from-literal openai-api-key=xxxxxxxx \
--dry-run=client -o yaml > caradoc-api-secret.yaml
$> make help | grep -i chart
charts-secrets create helm charts secrets
deploy-ext-charts deploy all external helm charts
deploy-app-charts deploy application helm charts (API & Web)
deploy-qdrant deploy qdrant helm chart
deploy-redis deploy redis helm chart
deploy-minio deploy minio helm chart
deploy-mongodb deploy mongodb helm chart
deploy-postgresql deploy postgresql helm chart (+ mlflow database creation)
deploy-mlflow deploy mlflow helm chart
deploy-caradoc-api deploy caradoc API helm chart
deploy-caradoc-web deploy caradoc Web helm chart
```
## Déploiement standard
Renseigner les valeurs par défaut pour l'environnement cible dans le fichier Makefile (namespace, ...).
### Créer les secrets K8s
```sh
$> make openai_api_key=<à définir> charts-secrets
```
### Déployer les charts des composants dont dépend Caradoc
Configurer/ajuster les valeurs des fichiers \*.values.yaml pour l'environnement cible avant les déploiements.
```sh
helm upgrade --install caradoc-api caradoc-api/ --reset-values -f dev-caradoc-api.values.yaml --namespace caradoc
$> make deploy-ext-charts
```
## caradoc-web
### Déployer les charts applicatifs (caradoc-api & caradoc-web)
Renseigner la valeur de l'URI public de MLflow dans le fichier \*.values.yaml avant le déploiement
Configurer/ajuster les valeurs (notamment les différents endpoints) des fichiers \*.values.yaml pour l'environnement cible avant les déploiements.
```sh
helm upgrade --install caradoc-web caradoc-web/ --reset-values -f dev-caradoc-web.values.yaml --namespace caradoc
$> make deploy-app-charts
```
NB : Il ne faut pas oublier de renseigner les différents endpoints dans les fichiers values de l'API et du Web
## Prérequis
- NFS provisioner pour la persistence
## Références
## qdrant
### qdrant
https://artifacthub.io/packages/helm/qdrant/qdrant
```sh
helm upgrade --install qdrant qdrant/ --reset-values -f dev-qdrant.values.yaml --namespace caradoc
```
## redis (bitnami)
### redis (bitnami)
https://artifacthub.io/packages/helm/bitnami/redis
```sh
kubectl create secret generic redis-secret \
--namespace caradoc \
--from-literal db-password=$(uuidgen -r) \
--dry-run=client -o yaml > redis-secret.yaml
```
```sh
helm upgrade --install redis redis/ --reset-values -f dev-redis.values.yaml --namespace caradoc
```
## minio (bitnami)
### minio (bitnami)
https://artifacthub.io/packages/helm/bitnami/minio
```sh
kubectl create secret generic minio-secret \
--namespace caradoc \
--from-literal minio-root-user=admin \
--from-literal minio-root-password=$(uuidgen -r) \
--dry-run=client -o yaml > minio-secret.yaml
```
```sh
helm upgrade --install minio minio/ --reset-values -f dev-minio.values.yaml --namespace caradoc
```
## mongodb (bitnami)
### mongodb (bitnami)
https://artifacthub.io/packages/helm/bitnami/mongodb
https://github.com/bitnami/containers/blob/main/bitnami/mongodb/README.md
```sh
export MONGODB_ROOT_USER=root && \
export MONGODB_ROOT_PASSWORD=$(uuidgen -r) && \
kubectl create secret generic mongodb-secret \
--namespace caradoc \
--from-literal mongodb-root-user=$MONGODB_ROOT_USER \
--from-literal mongodb-root-password=$MONGODB_ROOT_PASSWORD \
--from-literal mongodb-metrics-password=$(uuidgen -r) \
--from-literal mongodb-uri="mongodb://$MONGODB_ROOT_USER:$MONGODB_ROOT_PASSWORD@mongodb:27017/test?serverSelectionTimeoutMS=3000&authSource=admin" \
--dry-run=client -o yaml > mongodb-secret.yaml
```
```sh
helm upgrade --install mongodb mongodb/ --reset-values -f dev-mongodb.values.yaml --namespace caradoc
```
https://github.com/bitnami/containers/blob/main/bitnami/mongodb/README.md
## postgresql (bitnami)
### postgresql (bitnami)
https://artifacthub.io/packages/helm/bitnami/postgresql
```sh
kubectl create secret generic postgresql-secret \
--namespace caradoc \
--from-literal postgres-password=$(uuidgen -r) \
--from-literal password=$(uuidgen -r) \
--dry-run=client -o yaml > postgresql-secret.yaml
```
```sh
helm upgrade --install postgresql postgresql/ --reset-values -f dev-postgresql.values.yaml --namespace caradoc
```
### Créer la base de données mlflow
```sh
export POSTGRES_PASSWORD=$(kubectl get secret --namespace caradoc postgresql-secret -o jsonpath="{.data.postgres-password}" | base64 --decode)
kubectl run postgresql-client --rm --tty -i --restart='Never' \
--namespace caradoc \
--image docker.io/bitnami/postgresql:14.3.0-debian-10-r20 \
--env="PGPASSWORD=$POSTGRES_PASSWORD" \
--command -- psql --host postgresql -U postgres -d postgres -p 5432
CREATE DATABASE mlflow OWNER postgres;
```
## mlflow (bitnami)
### mlflow (bitnami)
https://artifacthub.io/packages/helm/bitnami/mlflow
```sh
kubectl create secret generic mlflow-secret \
--namespace caradoc \
--from-literal mlflow-user=admin \
--from-literal mlflow-password=$(uuidgen -r) \
--dry-run=client -o yaml > mlflow-secret.yaml
helm upgrade --install mlflow mlflow/ --reset-values -f dev-mlflow.values.yaml --namespace caradoc
```
{{- /*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{- if and .Values.tracking.enabled .Values.tracking.auth.enabled (not .Values.tracking.auth.existingSecret) }}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "mlflow.v0.tracking.fullname" . }}
namespace: {{ include "common.names.namespace" . | quote }}
labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
app.kubernetes.io/part-of: mlflow
app.kubernetes.io/component: tracking
{{- if .Values.commonAnnotations }}
annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }}
{{- end }}
data:
# We need to ad the username as it is required by the ServiceMonitor object
admin-user: {{ include "common.secrets.passwords.manage" (dict "secret" (include "mlflow.v0.tracking.fullname" .) "key" "admin-user" "providedValues" (list "tracking.auth.username") "context" $) }}
admin-password: {{ include "common.secrets.passwords.manage" (dict "secret" (include "mlflow.v0.tracking.fullname" .) "key" "admin-password" "providedValues" (list "tracking.auth.password") "context" $) }}
{{- end }}
{{- /*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{- if and .Values.tracking.enabled (not .Values.postgresql.enabled) (not .Values.externalDatabase.existingSecret) }}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "mlflow.v0.database.secretName" . }}
namespace: {{ include "common.names.namespace" . | quote }}
labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
app.kubernetes.io/part-of: mlflow
app.kubernetes.io/component: tracking
{{- if .Values.commonAnnotations }}
annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
{{- end }}
type: Opaque
data:
db-password: {{ .Values.externalDatabase.password | b64enc | quote }}
{{- end }}
{{- /*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{- if and
.Values.tracking.enabled
.Values.externalS3.useCredentialsInSecret
(not .Values.minio.enabled)
(not .Values.externalS3.existingSecret)
}}
apiVersion: v1
kind: Secret
metadata:
name: {{ include "mlflow.v0.s3.secretName" . }}
namespace: {{ include "common.names.namespace" . | quote }}
labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
app.kubernetes.io/part-of: mlflow
app.kubernetes.io/component: tracking
{{- if .Values.commonAnnotations }}
annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }}
{{- end }}
type: Opaque
data:
{{ .Values.externalS3.existingSecretAccessKeyIDKey }}: {{ .Values.externalS3.accessKeyID | b64enc | quote }}
{{ .Values.externalS3.existingSecretKeySecretKey }}: {{ .Values.externalS3.accessKeySecret | b64enc | quote }}
{{- end }}
{{- /*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{- if (include "mlflow.v0.tracking.createTlsSecret" . ) }}
{{- $secretName := printf "%s-crt" (include "mlflow.v0.tracking.fullname" .) }}
{{- $ca := genCA "mlflow-ca" 365 }}
{{- $fullname := include "mlflow.v0.tracking.fullname" . }}
{{- $releaseNamespace := include "common.names.namespace" . }}
{{- $clusterDomain := .Values.clusterDomain }}
{{- $altNames := list (printf "*.%s.%s.svc.%s" $fullname $releaseNamespace $clusterDomain) (printf "%s.%s.svc.%s" $fullname $releaseNamespace $clusterDomain) "127.0.0.1" $fullname }}
{{- $cert := genSignedCert $fullname nil $altNames 365 $ca }}
apiVersion: v1
kind: Secret
metadata:
name: {{ $secretName }}
namespace: {{ .Release.Namespace | quote }}
labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
app.kubernetes.io/part-of: mlflow
app.kubernetes.io/component: tracking
{{- if .Values.commonAnnotations }}
annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
{{- end }}
type: kubernetes.io/tls
data:
tls.crt: {{ include "common.secrets.lookup" (dict "secret" $secretName "key" "tls.crt" "defaultValue" $cert.Cert "context" $) }}
tls.key: {{ include "common.secrets.lookup" (dict "secret" $secretName "key" "tls.key" "defaultValue" $cert.Key "context" $) }}
ca.crt: {{ include "common.secrets.lookup" (dict "secret" $secretName "key" "ca.crt" "defaultValue" $ca.Cert "context" $) }}
{{- end }}
{{- /*
Copyright Broadcom, Inc. All Rights Reserved.
SPDX-License-Identifier: APACHE-2.0
*/}}
{{- if (include "redis.createTlsSecret" .) }}
{{- $secretName := printf "%s-crt" (include "common.names.fullname" .) }}
{{- $ca := genCA "redis-ca" 365 }}
{{- $releaseNamespace := (include "common.names.namespace" .) }}
{{- $clusterDomain := .Values.clusterDomain }}
{{- $fullname := include "common.names.fullname" . }}
{{- $serviceName := include "common.names.fullname" . }}
{{- $headlessServiceName := printf "%s-headless" (include "common.names.fullname" .) }}
{{- $masterServiceName := printf "%s-master" (include "common.names.fullname" .) }}
{{- $altNames := list (printf "*.%s.%s.svc.%s" $serviceName $releaseNamespace $clusterDomain) (printf "%s.%s.svc.%s" $masterServiceName $releaseNamespace $clusterDomain) (printf "*.%s.%s.svc.%s" $masterServiceName $releaseNamespace $clusterDomain) (printf "*.%s.%s.svc.%s" $headlessServiceName $releaseNamespace $clusterDomain) (printf "%s.%s.svc.%s" $headlessServiceName $releaseNamespace $clusterDomain) "127.0.0.1" "localhost" $fullname }}
{{- $cert := genSignedCert $fullname nil $altNames 365 $ca }}
apiVersion: v1
kind: Secret
metadata:
name: {{ $secretName }}
namespace: {{ include "common.names.namespace" . | quote }}
labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
{{- if .Values.commonAnnotations }}
annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
{{- end }}
type: kubernetes.io/tls
data:
tls.crt: {{ include "common.secrets.lookup" (dict "secret" $secretName "key" "tls.crt" "defaultValue" $cert.Cert "context" $) }}
tls.key: {{ include "common.secrets.lookup" (dict "secret" $secretName "key" "tls.key" "defaultValue" $cert.Key "context" $) }}
ca.crt: {{ include "common.secrets.lookup" (dict "secret" $secretName "key" "ca.crt" "defaultValue" $ca.Cert "context" $) }}
{{- end }}