# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
#
# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
# `get_provider_info_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
[docs]
def get_provider_info():
return {
"package-name": "apache-airflow-providers-apache-kafka",
"name": "Apache Kafka",
"description": "`Apache Kafka <https://kafka.apache.org/>`__\n",
"integrations": [
{
"integration-name": "Apache Kafka",
"external-doc-url": "https://kafka.apache.org/",
"how-to-guide": ["/docs/apache-airflow-providers-apache-kafka/sensors.rst"],
"logo": "/docs/integration-logos/kafka.svg",
"tags": ["apache"],
}
],
"operators": [
{
"integration-name": "Apache Kafka",
"python-modules": [
"airflow.providers.apache.kafka.operators.consume",
"airflow.providers.apache.kafka.operators.produce",
],
}
],
"asset-uris": [
{
"schemes": ["kafka"],
"handler": "airflow.providers.apache.kafka.assets.kafka.sanitize_uri",
"factory": "airflow.providers.apache.kafka.assets.kafka.create_asset",
"to_openlineage_converter": "airflow.providers.apache.kafka.assets.kafka.convert_asset_to_openlineage",
}
],
"dataset-uris": [
{
"schemes": ["kafka"],
"handler": "airflow.providers.apache.kafka.assets.kafka.sanitize_uri",
"factory": "airflow.providers.apache.kafka.assets.kafka.create_asset",
"to_openlineage_converter": "airflow.providers.apache.kafka.assets.kafka.convert_asset_to_openlineage",
}
],
"hooks": [
{
"integration-name": "Apache Kafka",
"python-modules": [
"airflow.providers.apache.kafka.hooks.base",
"airflow.providers.apache.kafka.hooks.client",
"airflow.providers.apache.kafka.hooks.consume",
"airflow.providers.apache.kafka.hooks.produce",
],
}
],
"sensors": [
{
"integration-name": "Apache Kafka",
"python-modules": ["airflow.providers.apache.kafka.sensors.kafka"],
}
],
"triggers": [
{
"integration-name": "Apache Kafka",
"python-modules": [
"airflow.providers.apache.kafka.triggers.await_message",
"airflow.providers.apache.kafka.triggers.msg_queue",
],
}
],
"connection-types": [
{
"hook-class-name": "airflow.providers.apache.kafka.hooks.base.KafkaBaseHook",
"hook-name": "Apache Kafka",
"connection-type": "kafka",
"ui-field-behaviour": {
"hidden-fields": ["schema", "login", "password", "port", "host"],
"relabeling": {"extra": "Config Dict"},
"placeholders": {
"extra": '{"bootstrap.servers": "localhost:9092", "group.id": "my-group"}'
},
},
}
],
"queues": ["airflow.providers.apache.kafka.queues.kafka.KafkaMessageQueueProvider"],
"plugins": [
{
"name": "kafka_event_producer",
"plugin-class": "airflow.providers.apache.kafka.plugins.event_producer.KafkaEventProducerPlugin",
}
],
"config": {
"kafka_event_producer": {
"description": "Settings for the Kafka event producer plugin that publishes Airflow\nDagRun and TaskInstance state-change events to a Kafka topic.\n",
"options": {
"dag_run_events_enabled": {
"description": "Publish DagRun state-change events (``dag_run.running``,\n``dag_run.success``, ``dag_run.failed``). When False the\nDagRun listener is not registered.\n",
"version_added": "1.14.1",
"type": "boolean",
"example": None,
"default": "False",
},
"task_instance_events_enabled": {
"description": "Publish TaskInstance state-change events (``task_instance.running``,\n``task_instance.success``, ``task_instance.failed``,\n``task_instance.skipped``). When False the TaskInstance listener\nis not registered.\n",
"version_added": "1.14.1",
"type": "boolean",
"example": None,
"default": "False",
},
"kafka_config_id": {
"description": "Airflow connection used to build the plugin's Kafka producer.\nWhen unset, the producer hook falls back to its default\nconnection (``kafka_default``).\n",
"version_added": "1.14.1",
"type": "string",
"example": "kafka_default",
"default": "",
},
"topic": {
"description": "Topic the plugin publishes events to. The topic must already\nexist on the broker; the plugin will not auto-create it.\n",
"version_added": "1.14.1",
"type": "string",
"example": None,
"default": "airflow.events",
},
"source": {
"description": "Identifier added to every emitted message under the ``source``\nfield so consumers can distinguish Airflow installations that\nshare the same topic. When unset, falls back to the hostname\nof the Airflow component that emits the event (scheduler,\nworker, etc.).\n",
"version_added": "1.14.1",
"type": "string",
"example": "af-prod-eu",
"default": "",
},
"dag_run_dag_id_allowlist": {
"description": "Comma-separated glob patterns. When set, DagRun events are only\nemitted for dag_ids matching at least one pattern. Empty = all dags.\n",
"version_added": "1.14.1",
"type": "string",
"example": "demo_*,test_dag1",
"default": "",
},
"dag_run_dag_id_denylist": {
"description": "Comma-separated glob patterns. DagRun events for dag_ids matching\nany pattern are skipped. Deny takes precedence over allow.\n",
"version_added": "1.14.1",
"type": "string",
"example": "demo_*",
"default": "",
},
"task_instance_dag_id_allowlist": {
"description": "Comma-separated glob patterns. When set, TaskInstance events are\nonly emitted for dag_ids matching at least one pattern.\nEmpty = all dags.\n",
"version_added": "1.14.1",
"type": "string",
"example": "demo_*,test_dag1",
"default": "",
},
"task_instance_dag_id_denylist": {
"description": "Comma-separated glob patterns. TaskInstance events for dag_ids\nmatching any pattern are skipped. Deny takes precedence over allow.\n",
"version_added": "1.14.1",
"type": "string",
"example": "demo_*",
"default": "",
},
"task_instance_task_id_allowlist": {
"description": "Comma-separated glob patterns. When set, TaskInstance events are\nonly emitted for task_ids matching at least one pattern. Applied\nin addition to ``task_instance_dag_id_allowlist`` — both must pass.\nMapped task instances share the same task_id so a single pattern\ncovers all map indices.\n",
"version_added": "1.14.1",
"type": "string",
"example": "load_*,extract_*",
"default": "",
},
"task_instance_task_id_denylist": {
"description": "Comma-separated glob patterns. TaskInstance events for task_ids\nmatching any pattern are skipped. Deny takes precedence over allow.\n",
"version_added": "1.14.1",
"type": "string",
"example": "*_cleanup",
"default": "",
},
"topic_check_timeout": {
"description": "How long (in seconds) each topic existence check is allowed\nto block waiting for a response from the broker.\n",
"version_added": "1.14.1",
"type": "integer",
"example": None,
"default": "10",
},
"topic_check_retry_interval": {
"description": "How long (in seconds) to wait before retrying a topic check,\nin case it failed.\n",
"version_added": "1.14.1",
"type": "integer",
"example": None,
"default": "60",
},
},
}
},
}