Apache Airflow Installation on Windows 10
############# Apachce Airflow 2.0 #############
# Githb repo to install pre-requisits :
https://github.com/tuanavu/airflow-tutorial
# Install Cloudera Quickstart VM
Launch HDP
#### Install Pyton3
$ sudo yum update
$ sudo yum install yum-utils
$ sudo yum install python36u
$ sudo yum install python36u-pip
# Create symlink
ln -s /usr/bin/python3.6 /usr/bin/python3
# create alias in ~/.bash_profile or ~/.bashrc
alias python3=’/usr/bin/python3.6'
source ~/.bashrc
source ~/.bash_profile
#### Install Docker :####
https://developpaper.com/offline-installation-of-docker-and-docker-compose-under-centos7/
#Docker CE requires some software packages that are available in EPEL (Extra Packages for Enterprise Linux) yum repository. Therefore, we have to install EPEL yum repository first.
yum install -y epel-release.noarch
# Install Docker CE yum repository on our CentOS 7 operating system.
yum-config-manager — add-repo=https://download.docker.com/linux/centos/docker-ce.repo
#Enable Docker CE (Nightly) yum repository.
yum-config-manager — enable docker-ce-nightly
#We have added two yum repositories in our Linux server. Therefore, we should build cache for yum package manager.
yum makecache fast
# Downloading Docker CE and Software Dependencies:
# Create a directory to download Docker CE and dependent software packages.
cd
cd downloads/
mkdir -p docker
cd docker
#Download Docker CE and dependent packages following Linux command.
yumdownloader — resolve docker-ce
### IMPORTANT : Now you can transfer zip and transfer these rpm files to any other server to install docker there.
#Start and enable Docker service.
sudo su
systemctl enable docker.service
for i in {0..6}
do
mknod -m0660 /dev/loop$i b 7 $i
done
sudo usermod -aG docker $(whoami)
sudo service docker stop
systemctl stop docker
sudo rm -rf /var/lib/docker
rm -rf /var/run/docker.pid
rm -rf /var/run/docker.pid
rm /var/run/docker.pid
rm -rf /var/snap/docker/179/run/docker.pid
ps axf | grep docker | grep -v grep | awk ‘{print “kill -9 “ $1}’ | sudo sh
systemctl reset-failed docker.service
#sudo service docker start (or systemctl start docker.service)
systemctl status docker
systemctl stop docker
#Execute below command
sudo dockerd
#Open a new terminal and perform below command to check if docker is running
docker version
#If any issue, execute belo command :
sudo dockerd
And then execute above 6 commands
### Upgrade Git Version ###
### Install DOcker Compose : ###
mkdir -p /root/downloads/docker_compose
#Download the package in a system with connectivity (replace the needed version as required)
wget https://github.com/docker/compose/releases/download/1.24.0/docker-compose-Linux-x86_64
#Just rename the package
mv docker-compose-Linux-x86_64 docker-compose
#Copy it (ssh) to the required system (To make it executable)
sudo mv docker-compose /usr/local/bin/
sudo chmod +x /usr/local/bin/docker-compose
docker-compose -v
#### Clone sample repo from tuanavu ####
mkdir /root/sarthak_workspace/docker_repo/
cd /root/sarthak_workspace/docker_repo/
git clone https://github.com/tuanavu/airflow-tutorial
cd /root/sarthak_workspace/docker_repo/airflow-tutorial
docker-compose up
or
docker-compose up -d
### If any issue, just update docker_config.yaml
You can use 5432 instead of 5432:5432 in your docker command or docker-compose file and let docker choose the host port automatically.
You can use 8080 instead of 8080:8080 in your docker command or docker-compose file and let docker choose the host port automatically.
# Displays log output :
docker-compose logs
#List containers
docker-compose ps
#Stop containers
docker-compose down
