# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Manage a Kubernetes Kueue."""
from __future__ import annotations
import json
import time
from collections.abc import Sequence
from functools import cached_property
from typing import TYPE_CHECKING, Any
from kubernetes.client import models as k8s
from kubernetes.client.rest import ApiException
from kubernetes.utils import FailToCreateError
from urllib3.exceptions import HTTPError
from airflow.providers.cncf.kubernetes.hooks.kubernetes import KubernetesHook
from airflow.providers.cncf.kubernetes.operators.job import KubernetesJobOperator
from airflow.providers.cncf.kubernetes.triggers.kueue import (
JOB_STATE_MESSAGES,
KubernetesKueueJobTrigger,
KueueJobState,
build_job_event,
collect_xcom_results,
get_job_state,
get_succeeded_pod_xcoms,
list_job_pods,
)
from airflow.providers.cncf.kubernetes.utils.pod_manager import XComRetrievalError
from airflow.providers.cncf.kubernetes.version_compat import AIRFLOW_V_3_1_PLUS
from airflow.providers.common.compat.sdk import AirflowException
if AIRFLOW_V_3_1_PLUS:
from airflow.sdk import BaseOperator
else:
from airflow.models import BaseOperator
# The ``error`` of a deferral timeout in Airflow 3 (``TriggerFailureReason.TRIGGER_TIMEOUT``) and in
# Airflow 2. Matched as text: Airflow 2 raises the generic ``TaskDeferralError`` for any trigger
# failure, and ``TaskDeferralTimeout`` moved to another module in Airflow 3.2.
_DEFERRAL_TIMEOUT_ERRORS = ("Trigger timeout", "Trigger/execution timeout")
if TYPE_CHECKING:
from pendulum import DateTime
from airflow.sdk import Context
[docs]
class KubernetesInstallKueueOperator(BaseOperator):
"""
Installs a Kubernetes Kueue.
.. seealso::
For more information on how to use this operator, take a look at the guide:
:ref:`howto/operator:KubernetesInstallKueueOperator`
:param kueue_version: The Kubernetes Kueue version to install.
:param kubernetes_conn_id: The :ref:`kubernetes connection id <howto/connection:kubernetes>`
for the Kubernetes cluster.
"""
[docs]
template_fields: Sequence[str] = (
"kueue_version",
"kubernetes_conn_id",
)
def __init__(self, kueue_version: str, kubernetes_conn_id: str = "kubernetes_default", *args, **kwargs):
super().__init__(*args, **kwargs)
[docs]
self.kubernetes_conn_id = kubernetes_conn_id
[docs]
self.kueue_version = kueue_version
@property
def _kueue_yaml_url(self) -> str:
return (
f"https://github.com/kubernetes-sigs/kueue/releases/download/{self.kueue_version}/manifests.yaml"
)
@cached_property
[docs]
def hook(self) -> KubernetesHook:
return KubernetesHook(conn_id=self.kubernetes_conn_id)
[docs]
def execute(self, context):
yaml_objects = self.hook.get_yaml_content_from_file(kueue_yaml_url=self._kueue_yaml_url)
try:
self.hook.apply_from_yaml_file(yaml_objects=yaml_objects)
except FailToCreateError as ex:
error_bodies = []
for e in ex.api_exceptions:
try:
if e.body:
error_bodies.append(json.loads(e.body))
else:
# If no body content, use reason as the message
reason = getattr(e, "reason", "Unknown")
error_bodies.append({"message": reason, "reason": reason})
except (json.JSONDecodeError, ValueError, TypeError):
# If the body is a string (e.g., in a 429 error), it can't be parsed as JSON.
# Use the body directly as the message instead.
error_bodies.append({"message": e.body, "reason": getattr(e, "reason", "Unknown")})
if next((e for e in error_bodies if e.get("reason") == "AlreadyExists"), None):
self.log.info("Kueue is already enabled for the cluster")
if errors := [e for e in error_bodies if e.get("reason") != "AlreadyExists"]:
error_message = "\n".join(
e.get("message") or e.get("body") or f"Unknown error: {e.get('reason', 'Unknown')}"
for e in errors
)
raise AirflowException(error_message)
return
self.hook.check_kueue_deployment_running(name="kueue-controller-manager", namespace="kueue-system")
self.log.info("Kueue installed successfully!")
[docs]
class KubernetesStartKueueJobOperator(KubernetesJobOperator):
"""
Executes a Kubernetes Job in Kueue.
With ``wait_until_job_complete=True``, the operator polls the Job itself rather than its pods: the Job
has no pods until Kueue admits it, and loses its running pods when Kueue preempts it.
.. seealso::
For more information on how to use this operator, take a look at the guide:
:ref:`howto/operator:KubernetesStartKueueJobOperator`
:param queue_name: The name of the Queue in the cluster
"""
[docs]
template_fields = tuple({"queue_name"} | set(KubernetesJobOperator.template_fields))
def __init__(self, queue_name: str, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
[docs]
self.queue_name = queue_name
if self.suspend is False:
raise AirflowException(
"The `suspend` parameter can't be False. If you want to use Kueue for running Job"
" in a Kubernetes cluster, set the `suspend` parameter to True.",
)
if self.suspend is None:
self.log.info(
"You have not set parameter `suspend` in class %s. "
"For running a Job in Kueue the `suspend` parameter has been set to True.",
self.__class__.__name__,
)
self.suspend = True
self.labels.update({"kueue.x-k8s.io/queue-name": self.queue_name})
self.annotations.update({"kueue.x-k8s.io/queue-name": self.queue_name})
[docs]
def execute(self, context: Context):
if not self.wait_until_job_complete:
return super().execute(context)
if self.parallelism is not None and self.parallelism < 1:
raise ValueError("parallelism cannot be less than 1 with `wait_until_job_complete=True`.")
self.name = self._set_name(self.name)
self.job_request_obj = self.build_job_request_obj(context)
self.job = self.create_job(job_request_obj=self.job_request_obj) # must set `self.job` for `on_kill`
ti = context["ti"]
ti.xcom_push(key="job_name", value=self.job.metadata.name)
ti.xcom_push(key="job_namespace", value=self.job.metadata.namespace)
if self.deferrable:
self.defer(
trigger=KubernetesKueueJobTrigger(
job_name=self.job.metadata.name,
job_namespace=self.job.metadata.namespace,
base_container_name=self.base_container_name,
kubernetes_conn_id=self.kubernetes_conn_id,
cluster_context=self.cluster_context,
config_file=self.config_file,
in_cluster=self.in_cluster,
poll_interval=self.job_poll_interval,
do_xcom_push=self.do_xcom_push,
),
method_name="execute_complete",
# Airflow 3 does not apply execution_timeout to the time spent deferred
timeout=self.execution_timeout,
)
return self.execute_complete(
context, self._wait_until_job_finished(self.job.metadata.name, self.job.metadata.namespace)
)
def _wait_until_job_finished(self, job_name: str, namespace: str) -> dict[str, Any]:
"""Poll the Job until it is finished, and return the event ``KubernetesKueueJobTrigger`` yields."""
state: KueueJobState | None = None
xcom_results: dict[str, str] = {}
while True:
job = self.hook.get_job_status(job_name=job_name, namespace=namespace)
if (new_state := get_job_state(job)) != state:
state = new_state
self.log.info("The job '%s' %s", job_name, JOB_STATE_MESSAGES[state])
if state == KueueJobState.FINISHED:
return build_job_event(job, xcom_results)
if state == KueueJobState.RUNNING and self.do_xcom_push:
collect_xcom_results(
self.client, self.pod_manager, job, self.base_container_name, xcom_results
)
time.sleep(self.job_poll_interval)
[docs]
def execute_complete(self, context: Context, event: dict, **kwargs):
if not isinstance(event.get("xcom_result"), dict):
# Only KubernetesKueueJobTrigger gives "xcom_result" as a dict (by pod name): this event is from
# KubernetesJobTrigger, for a task deferred before the provider upgrade
return super().execute_complete(context, event, **kwargs)
pods = sorted(
list_job_pods(
self.client,
event["namespace"],
event["pod_label_selector"],
event["job"]["metadata"]["uid"],
),
key=lambda pod: pod.metadata.name,
)
for pod in pods:
self.log_matching_pod(pod=pod, context=context)
if self.get_logs and event["status"] == "error":
# super() raises for a failed Job before it writes the logs of the pods
for pod in pods:
self._write_logs(pod)
xcom_result = get_succeeded_pod_xcoms(pods, event["xcom_result"])
result = super().execute_complete(
context,
{
**event,
"pod_names": [pod.metadata.name for pod in pods],
"pod_namespace": event["namespace"],
"xcom_result": xcom_result,
},
**kwargs,
)
# Only reached for a successful Job: super() raises for a failed one
succeeded = (event["job"].get("status") or {}).get("succeeded") or 0
if self.do_xcom_push and len(xcom_result) < succeeded:
raise XComRetrievalError(
f"Read the XComs of only {len(xcom_result)} of the {succeeded} pods that succeeded in "
f"the job '{event['name']}': the other pods were deleted, or their XComs could not be read"
)
return result
[docs]
def resume_execution(self, next_method: str, next_kwargs: dict[str, Any] | None, context: Context):
if next_method == "__fail__" and (next_kwargs or {}).get("error") in _DEFERRAL_TIMEOUT_ERRORS:
self._delete_job_after_deferral_timeout(context)
return super().resume_execution(next_method, next_kwargs, context)
def _delete_job_after_deferral_timeout(self, context: Context) -> None:
"""
Delete the Job when the deferral timed out, as ``on_kill`` does when the sync wait times out.
``on_kill`` is not called for a deferred task, and a Job that Kueue never admits would otherwise
stay in its queue, and be run if Kueue admits it later.
"""
ti = context["ti"]
# Without map_indexes, xcom_pull returns the values of every map index of a mapped task
name = ti.xcom_pull(task_ids=ti.task_id, key="job_name", map_indexes=ti.map_index)
namespace = ti.xcom_pull(task_ids=ti.task_id, key="job_namespace", map_indexes=ti.map_index)
if not name:
return
self.log.info("Deleting the job '%s' as the task timed out while waiting for it", name)
# on_kill deletes self.job, with on_kill_propagation_policy and termination_grace_period
self.job = k8s.V1Job(metadata=k8s.V1ObjectMeta(name=name, namespace=namespace))
try:
self.on_kill()
except ApiException as e:
self.log.warning("Could not delete the job '%s': %s", name, e)
def _write_logs(self, pod: k8s.V1Pod, follow: bool = False, since_time: DateTime | None = None) -> None:
"""Write the logs of the ``container_logs`` of a pod of the finished Job."""
# Failed pods of earlier attempts may have no readable logs left, e.g. once evicted
try:
self.pod_manager.fetch_requested_container_logs(pod=pod, containers=self.container_logs)
except (HTTPError, ApiException) as e:
self.log.warning("Could not read the logs of pod %s: %s", pod.metadata.name, e)