# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
#
# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
# `get_provider_info_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
[docs]
def get_provider_info():
return {
"package-name": "apache-airflow-providers-microsoft-azure",
"name": "Microsoft Azure",
"description": "`Microsoft Azure <https://azure.microsoft.com/>`__\n",
"integrations": [
{
"integration-name": "Microsoft Azure Batch",
"external-doc-url": "https://azure.microsoft.com/en-us/services/batch/",
"how-to-guide": ["/docs/apache-airflow-providers-microsoft-azure/operators/batch.rst"],
"logo": "/docs/integration-logos/Microsoft-Azure-Batch.png",
"tags": ["azure"],
},
{
"integration-name": "Microsoft Azure Blob Storage",
"external-doc-url": "https://azure.microsoft.com/en-us/services/storage/blobs/",
"how-to-guide": ["/docs/apache-airflow-providers-microsoft-azure/sensors/wasb_sensors.rst"],
"logo": "/docs/integration-logos/Blob-Storage.svg",
"tags": ["azure"],
},
{
"integration-name": "Microsoft Azure Container Instances",
"external-doc-url": "https://azure.microsoft.com/en-us/services/container-instances/",
"logo": "/docs/integration-logos/Container-Instances.svg",
"tags": ["azure"],
},
{
"integration-name": "Microsoft Azure Cosmos DB",
"external-doc-url": "https://azure.microsoft.com/en-us/services/cosmos-db/",
"how-to-guide": [
"/docs/apache-airflow-providers-microsoft-azure/sensors/cosmos_document_sensor.rst"
],
"logo": "/docs/integration-logos/Azure-Cosmos-DB.svg",
"tags": ["azure"],
},
{
"integration-name": "Microsoft Azure Data Explorer",
"external-doc-url": "https://azure.microsoft.com/en-us/services/data-explorer/",
"logo": "/docs/integration-logos/Microsoft-Azure-Data-Explorer.png",
"tags": ["azure"],
},
{
"integration-name": "Microsoft Azure Data Lake Storage",
"how-to-guide": ["/docs/apache-airflow-providers-microsoft-azure/operators/adls.rst"],
"external-doc-url": "https://azure.microsoft.com/en-us/services/storage/data-lake-storage/",
"logo": "/docs/integration-logos/Data-Lake-Storage.svg",
"tags": ["azure"],
},
{
"integration-name": "Microsoft Azure Files",
"external-doc-url": "https://azure.microsoft.com/en-us/services/storage/files/",
"logo": "/docs/integration-logos/Azure-Files.svg",
"tags": ["azure"],
},
{
"integration-name": "Microsoft Azure FileShare",
"external-doc-url": "https://cloud.google.com/storage/",
"logo": "/docs/integration-logos/Microsoft-Azure-Fileshare.png",
"tags": ["azure"],
},
{
"integration-name": "Microsoft Azure Data Factory",
"how-to-guide": [
"/docs/apache-airflow-providers-microsoft-azure/operators/adf_run_pipeline.rst"
],
"external-doc-url": "https://azure.microsoft.com/en-us/services/data-factory/",
"logo": "/docs/integration-logos/Azure-Data-Factory.svg",
"tags": ["azure"],
},
{
"integration-name": "Microsoft Azure",
"external-doc-url": "https://azure.microsoft.com/",
"logo": "/docs/integration-logos/Microsoft-Azure.png",
"tags": ["azure"],
},
{
"integration-name": "Microsoft Azure Service Bus",
"external-doc-url": "https://azure.microsoft.com/en-us/services/service-bus/",
"logo": "/docs/integration-logos/Service-Bus.svg",
"how-to-guide": ["/docs/apache-airflow-providers-microsoft-azure/operators/asb.rst"],
"tags": ["azure"],
},
{
"integration-name": "Microsoft Azure Synapse",
"external-doc-url": "https://azure.microsoft.com/en-us/services/synapse-analytics/",
"how-to-guide": [
"/docs/apache-airflow-providers-microsoft-azure/operators/azure_synapse.rst"
],
"tags": ["azure"],
},
{
"integration-name": "Microsoft Azure Data Lake Storage Client Gen2",
"external-doc-url": "https://azure.microsoft.com/en-us/products/storage/data-lake-storage/",
"logo": "/docs/integration-logos/Data-Lake-Storage.svg",
"tags": ["azure"],
},
{
"integration-name": "Microsoft Graph API",
"external-doc-url": "https://learn.microsoft.com/en-us/graph/use-the-api/",
"logo": "/docs/integration-logos/Microsoft-Graph-API.png",
"how-to-guide": [
"/docs/apache-airflow-providers-microsoft-azure/operators/msgraph.rst",
"/docs/apache-airflow-providers-microsoft-azure/sensors/msgraph.rst",
],
"tags": ["azure"],
},
{
"integration-name": "Microsoft Power BI",
"external-doc-url": "https://learn.microsoft.com/en-us/rest/api/power-bi/",
"how-to-guide": ["/docs/apache-airflow-providers-microsoft-azure/operators/powerbi.rst"],
"tags": ["azure"],
},
],
"operators": [
{
"integration-name": "Microsoft Azure Data Lake Storage",
"python-modules": ["airflow.providers.microsoft.azure.operators.adls"],
},
{
"integration-name": "Microsoft Azure Data Explorer",
"python-modules": ["airflow.providers.microsoft.azure.operators.adx"],
},
{
"integration-name": "Microsoft Azure Batch",
"python-modules": ["airflow.providers.microsoft.azure.operators.batch"],
},
{
"integration-name": "Microsoft Azure Container Instances",
"python-modules": ["airflow.providers.microsoft.azure.operators.container_instances"],
},
{
"integration-name": "Microsoft Azure Cosmos DB",
"python-modules": ["airflow.providers.microsoft.azure.operators.cosmos"],
},
{
"integration-name": "Microsoft Azure Blob Storage",
"python-modules": ["airflow.providers.microsoft.azure.operators.wasb_delete_blob"],
},
{
"integration-name": "Microsoft Azure Data Factory",
"python-modules": ["airflow.providers.microsoft.azure.operators.data_factory"],
},
{
"integration-name": "Microsoft Azure Service Bus",
"python-modules": ["airflow.providers.microsoft.azure.operators.asb"],
},
{
"integration-name": "Microsoft Azure Synapse",
"python-modules": ["airflow.providers.microsoft.azure.operators.synapse"],
},
{
"integration-name": "Microsoft Graph API",
"python-modules": ["airflow.providers.microsoft.azure.operators.msgraph"],
},
{
"integration-name": "Microsoft Power BI",
"python-modules": ["airflow.providers.microsoft.azure.operators.powerbi"],
},
],
"sensors": [
{
"integration-name": "Microsoft Azure Cosmos DB",
"python-modules": ["airflow.providers.microsoft.azure.sensors.cosmos"],
},
{
"integration-name": "Microsoft Azure Blob Storage",
"python-modules": ["airflow.providers.microsoft.azure.sensors.wasb"],
},
{
"integration-name": "Microsoft Azure Data Factory",
"python-modules": ["airflow.providers.microsoft.azure.sensors.data_factory"],
},
{
"integration-name": "Microsoft Graph API",
"python-modules": ["airflow.providers.microsoft.azure.sensors.msgraph"],
},
],
"filesystems": [
"airflow.providers.microsoft.azure.fs.adls",
"airflow.providers.microsoft.azure.fs.msgraph",
],
"hooks": [
{
"integration-name": "Microsoft Azure Container Instances",
"python-modules": [
"airflow.providers.microsoft.azure.hooks.container_volume",
"airflow.providers.microsoft.azure.hooks.container_registry",
"airflow.providers.microsoft.azure.hooks.container_instance",
],
},
{
"integration-name": "Microsoft Azure Data Explorer",
"python-modules": ["airflow.providers.microsoft.azure.hooks.adx"],
},
{
"integration-name": "Microsoft Azure FileShare",
"python-modules": ["airflow.providers.microsoft.azure.hooks.fileshare"],
},
{
"integration-name": "Microsoft Azure",
"python-modules": ["airflow.providers.microsoft.azure.hooks.base_azure"],
},
{
"integration-name": "Microsoft Azure Batch",
"python-modules": ["airflow.providers.microsoft.azure.hooks.batch"],
},
{
"integration-name": "Microsoft Azure Data Lake Storage",
"python-modules": ["airflow.providers.microsoft.azure.hooks.data_lake"],
},
{
"integration-name": "Microsoft Azure Cosmos DB",
"python-modules": ["airflow.providers.microsoft.azure.hooks.cosmos"],
},
{
"integration-name": "Microsoft Azure Blob Storage",
"python-modules": ["airflow.providers.microsoft.azure.hooks.wasb"],
},
{
"integration-name": "Microsoft Azure Data Factory",
"python-modules": ["airflow.providers.microsoft.azure.hooks.data_factory"],
},
{
"integration-name": "Microsoft Azure Service Bus",
"python-modules": ["airflow.providers.microsoft.azure.hooks.asb"],
},
{
"integration-name": "Microsoft Azure Data Lake Storage Client Gen2",
"python-modules": ["airflow.providers.microsoft.azure.hooks.data_lake"],
},
{
"integration-name": "Microsoft Azure Synapse",
"python-modules": ["airflow.providers.microsoft.azure.hooks.synapse"],
},
{
"integration-name": "Microsoft Graph API",
"python-modules": ["airflow.providers.microsoft.azure.hooks.msgraph"],
},
{
"integration-name": "Microsoft Power BI",
"python-modules": ["airflow.providers.microsoft.azure.hooks.powerbi"],
},
],
"triggers": [
{
"integration-name": "Microsoft Azure Data Factory",
"python-modules": ["airflow.providers.microsoft.azure.triggers.data_factory"],
},
{
"integration-name": "Microsoft Azure Blob Storage",
"python-modules": ["airflow.providers.microsoft.azure.triggers.wasb"],
},
{
"integration-name": "Microsoft Graph API",
"python-modules": ["airflow.providers.microsoft.azure.triggers.msgraph"],
},
{
"integration-name": "Microsoft Power BI",
"python-modules": ["airflow.providers.microsoft.azure.triggers.powerbi"],
},
{
"integration-name": "Microsoft Azure Service Bus",
"python-modules": ["airflow.providers.microsoft.azure.triggers.message_bus"],
},
],
"queues": ["airflow.providers.microsoft.azure.queues.asb.AzureServiceBusMessageQueueProvider"],
"transfers": [
{
"source-integration-name": "Local",
"target-integration-name": "Microsoft Azure Data Lake Storage",
"how-to-guide": "/docs/apache-airflow-providers-microsoft-azure/transfer/local_to_adls.rst",
"python-module": "airflow.providers.microsoft.azure.transfers.local_to_adls",
},
{
"source-integration-name": "Oracle",
"target-integration-name": "Microsoft Azure Data Lake Storage",
"python-module": "airflow.providers.microsoft.azure.transfers.oracle_to_azure_data_lake",
},
{
"source-integration-name": "Local",
"target-integration-name": "Microsoft Azure Blob Storage",
"how-to-guide": "/docs/apache-airflow-providers-microsoft-azure/transfer/local_to_wasb.rst",
"python-module": "airflow.providers.microsoft.azure.transfers.local_to_wasb",
},
{
"source-integration-name": "SSH File Transfer Protocol (SFTP)",
"target-integration-name": "Microsoft Azure Blob Storage",
"how-to-guide": "/docs/apache-airflow-providers-microsoft-azure/transfer/sftp_to_wasb.rst",
"python-module": "airflow.providers.microsoft.azure.transfers.sftp_to_wasb",
},
{
"source-integration-name": "Amazon Simple Storage Service (S3)",
"target-integration-name": "Microsoft Azure Blob Storage",
"how-to-guide": "/docs/apache-airflow-providers-microsoft-azure/transfer/s3_to_wasb.rst",
"python-module": "airflow.providers.microsoft.azure.transfers.s3_to_wasb",
},
],
"connection-types": [
{
"hook-class-name": "airflow.providers.microsoft.azure.hooks.base_azure.AzureBaseHook",
"connection-type": "azure",
"ui-field-behaviour": {
"hidden-fields": ["schema", "port", "host"],
"relabeling": {"login": "Azure Client ID", "password": "Azure Secret"},
"placeholders": {
"extra": '{"key_path": "path to json file for auth", "key_json": "specifies json dict for auth"}',
"login": "client_id (token credentials auth)",
"password": "secret (token credentials auth)",
"tenantId": "tenantId (token credentials auth)",
"subscriptionId": "subscriptionId (token credentials auth)",
},
},
},
{
"hook-class-name": "airflow.providers.microsoft.azure.hooks.adx.AzureDataExplorerHook",
"connection-type": "azure_data_explorer",
"ui-field-behaviour": {
"hidden-fields": ["schema", "port", "extra"],
"relabeling": {"login": "Username", "host": "Data Explorer Cluster URL"},
"placeholders": {
"login": "Varies with authentication method",
"password": "Varies with authentication method",
"auth_method": "AAD_APP/AAD_APP_CERT/AAD_CREDS/AAD_DEVICE/AZURE_TOKEN_CRED",
"tenant": "Used with AAD_APP/AAD_APP_CERT/AAD_CREDS",
"certificate": "Used with AAD_APP_CERT",
"thumbprint": "Used with AAD_APP_CERT",
},
},
},
{
"hook-class-name": "airflow.providers.microsoft.azure.hooks.batch.AzureBatchHook",
"connection-type": "azure_batch",
"ui-field-behaviour": {
"hidden-fields": ["schema", "port", "host", "extra"],
"relabeling": {"login": "Batch Account Name", "password": "Batch Account Access Key"},
"placeholders": {},
},
},
{
"hook-class-name": "airflow.providers.microsoft.azure.hooks.cosmos.AzureCosmosDBHook",
"connection-type": "azure_cosmos",
"ui-field-behaviour": {
"hidden-fields": ["schema", "port", "host", "extra"],
"relabeling": {"login": "Cosmos Endpoint URI", "password": "Cosmos Master Key Token"},
"placeholders": {
"login": "endpoint uri",
"password": "master key (not needed for Azure AD authentication)",
"database_name": "database name",
"collection_name": "collection name",
"subscription_id": "Subscription ID (required for Azure AD authentication)",
"resource_group_name": "Resource Group Name (required for Azure AD authentication)",
},
},
},
{
"hook-class-name": "airflow.providers.microsoft.azure.hooks.data_lake.AzureDataLakeHook",
"connection-type": "azure_data_lake",
"ui-field-behaviour": {
"hidden-fields": ["schema", "port", "host", "extra"],
"relabeling": {"login": "Azure Client ID", "password": "Azure Client Secret"},
"placeholders": {
"login": "client id",
"password": "secret",
"tenant": "tenant id",
"account_name": "datalake store",
},
},
},
{
"hook-class-name": "airflow.providers.microsoft.azure.hooks.fileshare.AzureFileShareHook",
"connection-type": "azure_fileshare",
"ui-field-behaviour": {
"hidden-fields": ["schema", "port", "host", "extra"],
"relabeling": {
"login": "Blob Storage Login (optional)",
"password": "Blob Storage Key (optional)",
},
"placeholders": {
"login": "account name or account url",
"password": "secret",
"sas_token": "account url or token (optional)",
"connection_string": "account url or token (optional)",
},
},
},
{
"hook-class-name": "airflow.providers.microsoft.azure.hooks.container_volume.AzureContainerVolumeHook",
"connection-type": "azure_container_volume",
"ui-field-behaviour": {
"hidden-fields": ["schema", "port", "host", "extra"],
"relabeling": {"login": "Azure Client ID", "password": "Azure Secret"},
"placeholders": {
"login": "client_id (token credentials auth)",
"password": "secret (token credentials auth)",
"connection_string": "connection string auth",
"subscription_id": "Subscription id (required for Azure AD authentication)",
"resource_group": "Resource group name (required for Azure AD authentication)",
},
},
},
{
"hook-class-name": "airflow.providers.microsoft.azure.hooks.container_instance.AzureContainerInstanceHook",
"connection-type": "azure_container_instance",
},
{
"hook-class-name": "airflow.providers.microsoft.azure.hooks.wasb.WasbHook",
"connection-type": "wasb",
"ui-field-behaviour": {
"hidden-fields": ["schema", "port"],
"relabeling": {
"login": "Blob Storage Login (optional)",
"password": "Blob Storage Key (optional)",
"host": "Account URL (Active Directory Auth)",
},
"placeholders": {
"login": "account name",
"password": "secret",
"host": "account url",
"connection_string": "connection string auth",
"tenant_id": "tenant",
"shared_access_key": "shared access key",
"sas_token": "account url or token",
"extra": "additional options for ClientSecretCredential or DefaultAzureCredential",
},
},
},
{
"hook-class-name": "airflow.providers.microsoft.azure.hooks.data_factory.AzureDataFactoryHook",
"connection-type": "azure_data_factory",
"ui-field-behaviour": {
"hidden-fields": ["schema", "port", "host", "extra"],
"relabeling": {"login": "Client ID", "password": "Secret"},
"placeholders": {},
},
},
{
"hook-class-name": "airflow.providers.microsoft.azure.hooks.container_registry.AzureContainerRegistryHook",
"connection-type": "azure_container_registry",
"ui-field-behaviour": {
"hidden-fields": ["schema", "port", "extra"],
"relabeling": {
"login": "Registry Username",
"password": "Registry Password",
"host": "Registry Server",
},
"placeholders": {
"login": "private registry username",
"password": "private registry password",
"host": "docker image registry server",
"subscription_id": "Subscription id (required for Azure AD authentication)",
"resource_group": "Resource group name (required for Azure AD authentication)",
},
},
},
{
"hook-class-name": "airflow.providers.microsoft.azure.hooks.asb.BaseAzureServiceBusHook",
"connection-type": "azure_service_bus",
"ui-field-behaviour": {
"hidden-fields": ["port", "host", "extra", "login", "password"],
"relabeling": {"schema": "Connection String"},
"placeholders": {
"fully_qualified_namespace": "<Resource group>.servicebus.windows.net (for Azure AD authentication)",
"credential": "credential",
"schema": "Endpoint=sb://<Resource group>.servicebus.windows.net/;SharedAccessKeyName=...",
},
},
},
{
"hook-class-name": "airflow.providers.microsoft.azure.hooks.synapse.BaseAzureSynapseHook",
"connection-type": "azure_synapse",
"ui-field-behaviour": {
"hidden-fields": ["schema", "port", "extra"],
"relabeling": {
"login": "Client ID",
"password": "Secret",
"host": "Synapse Workspace URL",
},
"placeholders": {},
},
},
{
"hook-class-name": "airflow.providers.microsoft.azure.hooks.data_lake.AzureDataLakeStorageV2Hook",
"connection-type": "adls",
"ui-field-behaviour": {
"hidden-fields": ["schema", "port"],
"relabeling": {
"login": "Client ID (Active Directory)",
"password": "ADLS Gen2 Key / Client Secret (Active Directory)",
"host": "ADLS Gen2 Account Name",
},
"placeholders": {
"extra": "additional options for use with FileService and AzureFileVolume",
"login": "client id",
"password": "key / secret",
"host": "storage account name",
"connection_string": "connection string (overrides auth)",
"tenant_id": "tenant id",
},
},
},
{
"hook-class-name": "airflow.providers.microsoft.azure.hooks.msgraph.KiotaRequestAdapterHook",
"connection-type": "msgraph",
"ui-field-behaviour": {
"hidden-fields": ["extra"],
"relabeling": {"login": "Client ID", "password": "Client Secret"},
"placeholders": {},
},
},
{
"hook-class-name": "airflow.providers.microsoft.azure.hooks.powerbi.PowerBIHook",
"connection-type": "powerbi",
"ui-field-behaviour": {
"hidden-fields": ["schema", "port", "host", "extra"],
"relabeling": {"login": "Client ID", "password": "Client Secret"},
"placeholders": {},
},
},
],
"secrets-backends": ["airflow.providers.microsoft.azure.secrets.key_vault.AzureKeyVaultBackend"],
"logging": ["airflow.providers.microsoft.azure.log.wasb_task_handler.WasbTaskHandler"],
"extra-links": [
"airflow.providers.microsoft.azure.operators.data_factory.AzureDataFactoryPipelineRunLink",
"airflow.providers.microsoft.azure.operators.synapse.AzureSynapsePipelineRunLink",
"airflow.providers.microsoft.azure.operators.powerbi.PowerBILink",
],
"config": {
"azure_remote_logging": {
"description": "Configuration that needs to be set for enable remote logging in Azure Blob Storage\n",
"options": {
"remote_wasb_log_container": {
"description": "WASB storage container where the remote logs will be stored.\nThe container should exist.\n",
"version_added": "8.0.0",
"type": "string",
"example": None,
"default": "airflow-logs",
}
},
}
},
}