curl -i -X PUT "http://[name-node-ip]:9870/jupyter_code/housing_data_etl.ipynb?op=CREATE&overwrite=true" \
  -T housing_data_etl.ipynb

hdfs dfs -cat hdfs://[name-node-ip]:9000/shared/housing_data/* | head -n 10

scp -r ubuntu@[server_address]:./mongo_backup ~/Documents/mongo_backup

scp [/path/to/local/file] ubuntu@[VPN Client IP]:/home/ubuntu/QM.pem

ssh ubuntu@[VPN Client IP]

ssh ubuntu@[server_address]

scp -r ~/Documents/mongo_backup ubuntu@[server_address]:./mongo_backup

sudo apt update
sudo apt install openjdk-17-jdk ssh -y

scp -r $HADOOP_HOME/etc/hadoop [data-node-ip]:$HADOOP_HOME/etc/

<configuration>
    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
    <property>
        <name>dfs.namenode.name.dir</name>
        <value>file:///usr/local/hadoop/data/namenode</value>
    </property>
    <property>
        <name>dfs.datanode.data.dir</name>
        <value>file:///usr/local/hadoop/data/datanode</value>
    </property>
    <property>
        <name>dfs.webhdfs.enabled</name>
        <value>true</value>
    </property>
</configuration>

sudo mkdir -p /usr/local/hadoop/data/namenode /usr/local/hadoop/data/datanode
sudo chown -R ubuntu:ubuntu /usr/local/hadoop/data

cat <<EOF > /usr/local/hadoop/etc/hadoop/core-site.xml
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://[name-node-ip]:9000</value>
    </property>
</configuration>
EOF

cat <<EOF > /usr/local/hadoop/etc/hadoop/hadoop-env.sh
#!/bin/bash
export JAVA_HOME=[java-home-path]
export HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop
export HADOOP_OS_TYPE=\$(uname -s)
export HADOOP_OPTS="\$HADOOP_OPTS --add-opens=java.base/java.lang=ALL-UNNAMED"
EOF

export JAVA_HOME=[java-home-path]
export HADOOP_HOME=/usr/local/hadoop
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

wget https://dlcdn.apache.org/hadoop/common/hadoop-3.4.3/hadoop-3.4.3-aarch64.tar.gz
sudo tar -xzvf hadoop-3.4.3-aarch64.tar.gz -C /usr/local/
sudo mv /usr/local/hadoop-3.4.3 /usr/local/hadoop
sudo chown -R ubuntu:ubuntu /usr/local/hadoop

ssh-keygen -t rsa -P "" -f ~/.ssh/id_rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 0600 ~/.ssh/authorized_keys
ssh-copy-id -i ~/.ssh/id_rsa.pub ubuntu@[data-node-ip]

#!/bin/bash
set -e

echo "=== CLEAN SLATE Hadoop setup START ==="

# ----------------------------
# 1. Delete old files (if exist)
# ----------------------------
sudo rm -f /usr/local/bin/hadoop-hosts-setup.sh
sudo rm -f /etc/systemd/system/hadoop-hosts.service

echo "✔ Old files removed"

# ----------------------------
# 2. Create fresh host script
# ----------------------------
sudo tee /usr/local/bin/hadoop-hosts-setup.sh > /dev/null <<'EOF'
#!/bin/bash

HOSTS_FILE="/etc/hosts"

# you can list more hosts here
HOSTS=(
[host-lists]
)

# Remove all old entries dynamically
for entry in "${HOSTS[@]}"; do
    ip=$(echo "$entry" | awk '{print $1}')
    hostname=$(echo "$entry" | awk '{print $2}')
    sudo sed -i "/$ip $hostname/d" "$HOSTS_FILE"
done

# Append fresh clean block (single source of truth)
{
for entry in "${HOSTS[@]}"; do
    echo "$entry"
done
} | sudo tee -a "$HOSTS_FILE" > /dev/null

EOF

sudo chmod +x /usr/local/bin/hadoop-hosts-setup.sh

echo "✔ New host script created"

# ----------------------------
# 3. Create fresh systemd service
# ----------------------------
sudo tee /etc/systemd/system/hadoop-hosts.service > /dev/null <<EOF
[Unit]
Description=Hadoop Hosts Setup (Clean Slate)
After=network.target

[Service]
Type=oneshot
ExecStart=/usr/local/bin/hadoop-hosts-setup.sh

[Install]
WantedBy=multi-user.target
EOF

echo "✔ New systemd service created"

# ----------------------------
# 4. Reload systemd + enable
# ----------------------------
sudo systemctl daemon-reload
sudo systemctl enable hadoop-hosts.service
sudo systemctl restart hadoop-hosts.service

echo "=== CLEAN SLATE Hadoop setup DONE ==="

cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF

cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
EOF

sudo apt install -y ca-certificates curl gpg
sudo mkdir -p /etc/apt/keyrings
curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.35/deb/Release.key \
| sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.35/deb/ /" \
| sudo tee /etc/apt/sources.list.d/kubernetes.list

sudo apt update
sudo apt install --reinstall linux-image-raspi linux-headers-raspi

sudo apt update
sudo apt install -y kubelet kubeadm kubectl
sudo apt-mark hold kubelet kubeadm kubectl

sudo apt update
sudo apt install -y containerd
sudo mkdir -p /etc/containerd
sudo containerd config default | sudo tee /etc/containerd/config.toml
sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' \
/etc/containerd/config.toml
sudo systemctl restart containerd
sudo systemctl enable containerd
containerd --version

sudo rm -rf \
  /etc/kubernetes \
  /var/lib/etcd \
  /var/lib/kubelet \
  /etc/cni/net.d \
  /var/lib/cni \
  ~/.kube

sudo iptables -F
sudo iptables -t nat -F
sudo iptables -t mangle -F
sudo iptables -X

apiVersion: kubeadm.k8s.io/v1beta4
kind: InitConfiguration
nodeRegistration:
  criSocket: unix:///run/containerd/containerd.sock
  kubeletExtraArgs:
    - name: feature-gates
      value: "NodeSwap=true"
    - name: fail-swap-on
      value: "false"
    - name: node-status-update-frequency
      value: "1m"

---
apiVersion: kubeadm.k8s.io/v1beta4
kind: ClusterConfiguration
networking:
  podSubnet: "10.244.0.0/16"

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

- name: IP_AUTODETECTION_METHOD
          value: "interface=^wlan0$"

sudo kubeadm reset -f
sudo rm -rf /var/lib/etcd /etc/cni/net.d /var/lib/cni ~/.kube

sudo modprobe overlay
sudo modprobe br_netfilter

docker() {
    if [ "$1" = "ps" ]; then
        command docker ps --format 'table {{.ID}}\t{{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}\t{{.CreatedAt}}'
    else
        command docker "$@"
    fi
}

sudo rm -rf /etc/cni/net.d/*
sudo systemctl restart containerd kubelet

sudo systemctl daemon-reload
sudo systemctl restart wg_check.service
sudo systemctl enable --now wg_check.timer
sudo systemctl restart wg_check.timer

[Unit]
Description=Run WireGuard handshake check every 1 minute

[Timer]
OnBootSec=10s      
OnUnitActiveSec=60s
AccuracySec=1s
Unit=wg_check.service

[Install]
WantedBy=timers.target

sudo tee /etc/crictl.yaml >/dev/null <<EOF
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 10
debug: false
EOF

etcd
  ↓
kube-apiserver
  ↓
kube-scheduler
kube-controller-manager
kube-proxy

curl -k -H "Authorization: Bearer <TOKEN>" \
     http://:6443/api/v1/namespaces/default/services

[Unit]
Description=Run WireGuard handshake check

[Service]
Type=oneshot
WorkingDirectory=/home/ubuntu
User=root
ExecStart=/bin/bash -c '/home/ubuntu/wg_handshake_check.sh 2>&1 | tee -a /home/ubuntu/wg_check.log;'

/home/ubuntu/wg_check.log {
    daily
    rotate 3
    maxage 3
    missingok
    notifempty
    create 0644 ubuntu ubuntu
}

#!/bin/bash
RED='\033[0;31m'          # Red
GREEN='\033[0;32m'        # Green
YELLOW='\033[0;33m'       # Yellow
RM_BOLD='\033[0m'        # Remove Bold
NC='\033[0m' # No Color
PEER_IP="10.0.0.1"
_VPN_SERVER_ADDR="[vpn-server-ip]"
_VPN_CLIENT_NAME="rspi58gb"

sudo apt-get update
sudo apt-get install -y wireguard

if ping -c 1 -W 1 "$PEER_IP" >/dev/null 2>&1; then
        echo -e "${GREEN}✓$(date) - VPN OK:  reachable${NC}"
else
        echo -e "${YELLOW}$(date) - VPN is Down: Trying to establish the connection with $PEER_IP.${NC}"
        wg genkey | tee /home/ubuntu/client_private.key | wg pubkey > /home/ubuntu/client_public.key
        sudo chmod 400 /home/ubuntu/QM.pem
        conf_fileName="${_VPN_CLIENT_NAME}.conf"
cat <<EOF > /home/ubuntu/$conf_fileName
[Interface]
PrivateKey = $(sudo ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /home/ubuntu/QM.pem ubuntu@${_VPN_SERVER_ADDR} sudo cat /home/ubuntu/server_private.key | xargs)
Address = 10.0.0.1/24
ListenPort = 51820
MTU = 1280

[Peer]
PublicKey = $(sudo cat /home/ubuntu/client_public.key | xargs)
AllowedIPs = 10.0.0.2/32
EOF
sudo scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /home/ubuntu/QM.pem $conf_fileName ubuntu@${_VPN_SERVER_ADDR}:/tmp/
sudo ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /home/ubuntu/QM.pem ubuntu@${_VPN_SERVER_ADDR} \
"sudo mv /tmp/$conf_fileName /etc/wireguard/ && sudo chown root:root /etc/wireguard/$conf_fileName && sudo chmod 600 /etc/wireguard/$conf_fileName"
sudo ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /home/ubuntu/QM.pem ubuntu@${_VPN_SERVER_ADDR} "sudo systemctl daemon-reload"
sudo ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /home/ubuntu/QM.pem ubuntu@${_VPN_SERVER_ADDR} "sudo systemctl enable wg-quick@$_VPN_CLIENT_NAME"
sudo ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /home/ubuntu/QM.pem ubuntu@${_VPN_SERVER_ADDR} "sudo systemctl restart wg-quick@$_VPN_CLIENT_NAME"
sudo ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /home/ubuntu/QM.pem ubuntu@${_VPN_SERVER_ADDR} "sudo sysctl -w net.ipv4.ip_forward=1
echo 'net.ipv4.ip_forward=1' | sudo tee -a /etc/sysctl.conf
sudo sysctl -p

# for handshaking of Masquerade all VPN tunnel traffic
sudo iptables -t nat -A POSTROUTING -o ${_VPN_CLIENT_NAME} -j MASQUERADE

# Forward TCP ports except 22 (SSH) and 88 (python server) to vpn
sudo iptables -t nat -A PREROUTING -i ens5 -p tcp -m multiport ! --dports [vpn-server-port-not-to-forward] -j DNAT --to-destination 10.0.0.2

# Forward UDP ports except 51820 (WireGuard) to vpn
sudo iptables -t nat -A PREROUTING -i ens5 -p udp ! --dport 51820 -j DNAT --to-destination 10.0.0.2

# alloq all Masquerade return traffic from vpn to browser
sudo iptables -t nat -A POSTROUTING -s 10.0.0.2/32 -o ens5 -j MASQUERADE"
sudo cat <<EOF > /tmp/vpn_client.conf
[Interface]
PrivateKey = $(sudo cat /home/ubuntu/client_private.key | xargs)
Address = 10.0.0.2/24
MTU = 1280

[Peer]
# EC2
PublicKey = $(sudo ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /home/ubuntu/QM.pem ubuntu@${_VPN_SERVER_ADDR} sudo cat /home/ubuntu/server_public.key | xargs)
Endpoint = ${_VPN_SERVER_ADDR}:51820
AllowedIPs = 10.0.0.0/24, 192.168.1.0/24, fd00::/64
PersistentKeepalive = 25
EOF
sudo mv /tmp/vpn_client.conf /etc/wireguard/ && sudo chown root:root /etc/wireguard/vpn_client.conf && sudo chmod 600 /etc/wireguard/vpn_client.conf
sudo systemctl daemon-reload
sudo systemctl enable wg-quick@vpn_client
sudo systemctl restart wg-quick@vpn_client
fi

sudo systemctl daemon-reload
sudo systemctl enable wg-quick@vpn_client
sudo systemctl restart wg-quick@vpn_client

sudo cat <<EOF > /tmp/vpn_client.conf
[Interface]
PrivateKey = $(sudo cat /home/ubuntu/client_private.key | xargs)
Address = 10.0.0.2/24
MTU = 1280

[Peer]
# EC2
PublicKey = $(sudo ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /home/ubuntu/QM.pem ubuntu@${_VPN_SERVER_ADDR} sudo cat /home/ubuntu/server_public.key | xargs)
Endpoint = ${_VPN_SERVER_ADDR}:51820
AllowedIPs = 10.0.0.0/24, 192.168.1.0/24, fd00::/64
PersistentKeepalive = 25
EOF

sudo ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /home/ubuntu/QM.pem ubuntu@${_VPN_SERVER_ADDR} "sudo sysctl -w net.ipv4.ip_forward=1
echo 'net.ipv4.ip_forward=1' | sudo tee -a /etc/sysctl.conf
sudo sysctl -p

# for handshaking of Masquerade all VPN tunnel traffic
sudo iptables -t nat -A POSTROUTING -o ${_VPN_CLIENT_NAME} -j MASQUERADE

# Forward TCP ports except 22 (SSH) and 88 (python server) to vpn
sudo iptables -t nat -A PREROUTING -i ens5 -p tcp -m multiport ! --dports [vpn-server-port-not-to-forward] -j DNAT --to-destination 10.0.0.2

# Forward UDP ports except 51820 (WireGuard) to vpn
sudo iptables -t nat -A PREROUTING -i ens5 -p udp ! --dport 51820 -j DNAT --to-destination 10.0.0.2

# alloq all Masquerade return traffic from vpn to browser
sudo iptables -t nat -A POSTROUTING -s 10.0.0.2/32 -o ens5 -j MASQUERADE"

sudo ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /home/ubuntu/QM.pem ubuntu@${_VPN_SERVER_ADDR} "sudo systemctl daemon-reload"
sudo ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /home/ubuntu/QM.pem ubuntu@${_VPN_SERVER_ADDR} "sudo systemctl enable wg-quick@$_VPN_CLIENT_NAME"
sudo ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /home/ubuntu/QM.pem ubuntu@${_VPN_SERVER_ADDR} "sudo systemctl restart wg-quick@$_VPN_CLIENT_NAME"

sudo ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /home/ubuntu/QM.pem ubuntu@${_VPN_SERVER_ADDR} \
"sudo mv /tmp/$conf_fileName /etc/wireguard/ && sudo chown root:root /etc/wireguard/$conf_fileName && sudo chmod 600 /etc/wireguard/$conf_fileName"

conf_fileName="${_VPN_CLIENT_NAME}.conf"
cat <<EOF > /home/ubuntu/$conf_fileName
[Interface]
PrivateKey = $(sudo ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /home/ubuntu/QM.pem ubuntu@${_VPN_SERVER_ADDR} sudo cat /home/ubuntu/server_private.key | xargs)
Address = 10.0.0.1/24
ListenPort = 51820
MTU = 1280

[Peer]
PublicKey = $(sudo cat /home/ubuntu/client_public.key | xargs)
AllowedIPs = 10.0.0.2/32
EOF

sudo apt-get update
sudo apt-get install -y wireguard

grep -q '^export _VPN_CLIENT_NAME=' ~/.bashrc && \
sed -i 's/^export _VPN_CLIENT_NAME=.*/export _VPN_CLIENT_NAME="[VPN Client Name Only with alphanumeric]"/' ~/.bashrc || \
echo 'export _VPN_CLIENT_NAME="[VPN Client Name Only with alphanumeric]"' >> ~/.bashrc

grep -q '^export _VPN_SERVER_ADDR=' ~/.bashrc && \
sed -i 's/^export _VPN_SERVER_ADDR=.*/export _VPN_SERVER_ADDR="[VPN Server IP]"/' ~/.bashrc || \
echo 'export _VPN_SERVER_ADDR="[VPN Server IP]"' >> ~/.bashrc

sudo apt update
sudo apt install --reinstall linux-image-raspi linux-headers-raspi

sudo tee /etc/systemd/resolved.conf.d/adguardhome.conf <<EOF
[Resolve]
DNS=127.0.0.1
DNSStubListener=no
EOF

network:
  version: 2
  wifis:
    wlan0:
      optional: true
      dhcp4: false
      addresses:
        - [server-static-ip]/24
      gateway4: 192.168.0.1
      nameservers:
        addresses:
          - 192.168.0.4
          - 192.168.0.1
      regulatory-domain: "US"
      access-points:
        "Wifi Name":
          auth:
            key-management: "psk"
            password: "encrypted password"

sudo nano /etc/systemd/resolved.conf

Make only changes to

[Resolve]
DNSStubListener=no