본문 바로가기

카테고리 없음

Airflow DB Cleanup Cronjob

apiVersion: batch/v1
kind: CronJob
metadata:
  labels:
    chart: airflow-1.20.0
    component: database-cleanup
    heritage: Helm
    release: airflow
    tier: airflow
  name: airflow-database-cleanup
  namespace: airflow
spec:
  concurrencyPolicy: Forbid
  failedJobsHistoryLimit: 1
  jobTemplate:
    metadata: {}
    spec:
      backoffLimit: 1
      template:
        metadata:
          labels:
            component: database-cleanup
            release: airflow
            tier: airflow
        spec:
          affinity: {}
          containers:
          - args:
            - -c
            - CLEAN_TS=$(date -d "-1 days" +"%Y-%m-%dT%H:%M:%S"); echo "Cleaning up
              metadata DB entries older than ${CLEAN_TS}"; exec airflow db clean --clean-before-timestamp
              "${CLEAN_TS}" --yes --verbose
            command:
            - bash
            env:
            - name: AIRFLOW__CORE__FERNET_KEY
              valueFrom:
                secretKeyRef:
                  key: fernet-key
                  name: airflow-fernet-key
            - name: AIRFLOW_HOME
              value: /opt/airflow
            - name: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN
              valueFrom:
                secretKeyRef:
                  key: connection
                  name: airflow-metadata
            - name: AIRFLOW_CONN_AIRFLOW_DB
              valueFrom:
                secretKeyRef:
                  key: connection
                  name: airflow-metadata
            - name: AIRFLOW__API__SECRET_KEY
              valueFrom:
                secretKeyRef:
                  key: api-secret-key
                  name: airflow-api-secret-key
            - name: AIRFLOW__CELERY__BROKER_URL
              valueFrom:
                secretKeyRef:
                  key: connection
                  name: airflow-broker-url
            image: apache/airflow:3.1.8
            imagePullPolicy: IfNotPresent
            name: database-cleanup
            resources: {}
            securityContext:
              allowPrivilegeEscalation: false
              capabilities:
                drop:
                - ALL
            terminationMessagePath: /dev/termination-log
            terminationMessagePolicy: File
            volumeMounts:
            - mountPath: /opt/airflow/airflow.cfg
              name: config
              readOnly: true
              subPath: airflow.cfg
            - mountPath: /opt/airflow/config/airflow_local_settings.py
              name: config
              readOnly: true
              subPath: airflow_local_settings.py
          dnsPolicy: ClusterFirst
          restartPolicy: Never
          schedulerName: default-scheduler
          securityContext:
            fsGroup: 0
            runAsUser: 50000
          serviceAccount: airflow-database-cleanup
          serviceAccountName: airflow-database-cleanup
          terminationGracePeriodSeconds: 30
          volumes:
          - configMap:
              defaultMode: 420
              name: airflow-config
            name: config
  schedule: 0 0 * * *
  successfulJobsHistoryLimit: 1
  suspend: false