| FROM quay.io/astronomer/astro-runtime:6.0.4 | |
| USER root | |
| RUN apt-get update && apt-get install -y patch patchutils && pip install watchdog~=2.1.9 | |
| # Resources: | |
| # - https://man7.org/linux/man-pages/man1/patch.1.html | |
| # - https://stackoverflow.com/a/543045/3066428 | |
| # - https://github.com/apache/airflow/blob/2.4.3/airflow/utils/file.py |
| import logging | |
| import random | |
| import string | |
| import threading | |
| import time | |
| from collections import defaultdict | |
| from dataclasses import dataclass | |
| from typing import Dict, Set | |
| logging.basicConfig(level=logging.INFO, format="%(threadName)s %(message)s") |
| import docker | |
| # fmt: off | |
| backport_package_class_mapping = { | |
| "apache-airflow-backport-providers-amazon==2020.5.20rc2": ["airflow.providers.amazon.aws.hooks.athena.AWSAthenaHook", "airflow.providers.amazon.aws.hooks.aws_dynamodb.AwsDynamoDBHook", "airflow.providers.amazon.aws.hooks.base_aws.AwsBaseHook", "airflow.providers.amazon.aws.hooks.cloud_formation.AWSCloudFormationHook", "airflow.providers.amazon.aws.hooks.datasync.AWSDataSyncHook", "airflow.providers.amazon.aws.hooks.ec2.EC2Hook", "airflow.providers.amazon.aws.hooks.emr.EmrHook", "airflow.providers.amazon.aws.hooks.glue.AwsGlueJobHook", "airflow.providers.amazon.aws.hooks.glue_catalog.AwsGlueCatalogHook", "airflow.providers.amazon.aws.hooks.kinesis.AwsFirehoseHook", "airflow.providers.amazon.aws.hooks.lambda_function.AwsLambdaHook", "airflow.providers.amazon.aws.hooks.logs.AwsLogsHook", "airflow.providers.amazon.aws.hooks.redshift.RedshiftHook", "airflow.providers.amazon.aws.hooks.s3.S3Hook", "airflow.providers.amazon.aws.hooks.sagemaker.SageMaker |
| {"name":"[DAG]","children":[{"name":"runme_0","children":[{"name":"run_after_loop","children":[{"name":"run_this_last"}]}]},{"name":"runme_1","children":[{"name":"run_after_loop"}]},{"name":"runme_2","children":[{"name":"run_after_loop"}]},{"name":"also_run_this","children":[{"name":"run_this_last"}]}]} |
| """Test the validity of all DAGs.""" | |
| import glob | |
| from os import path | |
| import pytest | |
| from airflow import models as airflow_models | |
| DAG_PATHS = glob.glob(path.join(path.dirname(__file__), "..", "..", "dags", "*.py")) | |
To set up the development environment:
docker-compose up -d postgresdocker-compose up -d airflow37 # or other versiondocker logs -f airflow_airflow37_1docker exec -it airflow_airflow37_1 bashairflow webserver
| #!/usr/bin/env bash | |
| set -x | |
| black=$1 | |
| input_file=$2 | |
| start_line=$3 | |
| end_line=$4 | |
| # Read selected lines and write to tmpfile |
| docker-ssh() { | |
| docker exec -i -t $1 sh | |
| } | |
| docker-update-time() { | |
| docker run --rm --privileged --pid=host walkerlee/nsenter -t 1 -m -u -i -n ntpd -d -q -n -p `cat /etc/ntp.conf | awk '{ print $2 }'` | |
| } | |
| docker-cleanup-volumes() { | |
| docker volume rm `docker volume ls -q -f dangling=true` |