tests.system.amazon.aws.example_glue

Module Contents

Functions

get_role_name(arn)

glue_cleanup(crawler_name, job_name, db_name)

Attributes

DAG_ID

ROLE_ARN_KEY

sys_test_context_task

EXAMPLE_CSV

EXAMPLE_SCRIPT

test_context

test_run

tests.system.amazon.aws.example_glue.DAG_ID = 'example_glue'[source]
tests.system.amazon.aws.example_glue.ROLE_ARN_KEY = 'ROLE_ARN'[source]
tests.system.amazon.aws.example_glue.sys_test_context_task[source]
tests.system.amazon.aws.example_glue.EXAMPLE_CSV = Multiline-String[source]
Show Value
"""product,value
apple,0.5
milk,2.5
bread,4.0
"""
tests.system.amazon.aws.example_glue.EXAMPLE_SCRIPT = Multiline-String[source]
Show Value
"""
from pyspark.context import SparkContext
from awsglue.context import GlueContext

glueContext = GlueContext(SparkContext.getOrCreate())
datasource = glueContext.create_dynamic_frame.from_catalog(
             database='{db_name}', table_name='input')
print('There are %s items in the table' % datasource.count())

datasource.toDF().write.format('csv').mode("append").save('s3://{bucket_name}/output')
"""
tests.system.amazon.aws.example_glue.get_role_name(arn)[source]
tests.system.amazon.aws.example_glue.glue_cleanup(crawler_name, job_name, db_name)[source]
tests.system.amazon.aws.example_glue.test_context[source]
tests.system.amazon.aws.example_glue.test_run[source]

Was this entry helpful?