[Apache Airflow] Airflow - Docker Ubuntu18.04
Airflow - Docker Ubuntu18.04
- Run Docker Ubuntu 18.04 Image
- Install Python
- Airflow Install & Configuration
- Commit & Run Docker Container
Run Docker Ubuntu 18.04 Image
- Run Docker Ubuntu 18.04 Image
docker container run -it -p 8090:8080 -e LC_ALL=C.UTF-8 --name airflow ubuntu:18.04
Install Python
# install needed packages
apt-get update -y
apt-get install -y curl
apt-get install -y build-essential
apt-get install -y zlib1g-dev
apt-get install -y libncurses5-dev
apt-get install -y libgdbm-dev
apt-get install -y libnss3-dev
apt-get install -y libssl-dev
apt-get install -y libreadline-dev
apt-get install -y libffi-dev
apt-get install -y libbz2-dev
apt-get install libsqlite3-dev
# install python
PYTHON_VERSION=3.8.10
PYTHON_VERSION_PREFIX="$(echo $PYTHON_VERSION | cut -d "." -f 1-2)"
cd tmp
curl -O https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz
tar -zxvf Python-${PYTHON_VERSION}.tgz
cd Python-${PYTHON_VERSION}
./configure --enable-optimizations
make altinstall
ln -s /usr/local/bin/${PYTHON_VERSION_PREFIX} /usr/local/bin/python3
curl -O https://bootstrap.pypa.io/get-pip.py
python3 get-pip.py
rm -rf ../*
rm -rf /var/lib/apt/lists/*
Airflow Install & Configuration
# create user
adduser airflow
# password (enter)
airflow
# create & activate virtual environment
mkdir airflow
cd airflow
python3 -m venv .venv
source ./.venv/bin/activate
# install airflow
pip3 install --upgrade pip
pip3 install --upgrade setuptools
pip3 install --upgrade distlib
AIRFLOW_HOME=$(pwd)
AIRFLOW_VERSION=2.1.4
PYTHON_VERSION="$(python3 --version | cut -d " " -f 2 | cut -d "." -f 1-2)"
CONSTRAINT_URL="https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-${PYTHON_VERSION}.txt"
pip3 install "apache-airflow==${AIRFLOW_VERSION}" --constraint "${CONSTRAINT_URL}"
# add in user airflow's .bashrc
export AIRFLOW_HOME=/home/airflow/airflow
# initialize meta store meta database needed by airflow
airflow db init
Commit & Run Docker Container
- DB를 postgres로 바꿔주는건 🔗 링크 참고
- 완료 후 나머지는 local의 zsh에서 진행
docker run -p 5432:5432 --name postgres1 -e POSTGRES_PASSWORD=1234 -d -v psql_data:/var/lib/postgresql/data postgres:13
# create airflownet and include postgres in the airflownet
docker network create airflownet
docker network connect airflownet postgres1
# commit container as image
docker commit airflow zsu58/ubuntu-airflow:18.04-2.1.4-psql
# execute command in the directory 'dags' (for bind-mount)
docker container run -it -p 8090:8080 --network airflownet -v $(pwd):/home/airflow/airflow/dags -e LC_ALL=C.UTF-8 --name airflow zsu58/ubuntu-airflow:18.04-2.1.4-psql
# 시작
docker container start airflow
docker container exec -it airflow bash