Arcade Voyage: Language, Voice, and Multimodal Processing | GSP104

GSP104 — Managed Service for Apache Spark: Qwik Start - Command Line

Estimated time: 15 minutes

# 🚀 Managed Service for Apache Spark: Qwik Start - Command Line | GSP104 > ⚠️ **Disclaimer:** This is an independent, community-made walkthrough created for educational purposes, hands-on practice, and Google Cloud certification preparation. This guide is designed to help learners understand Managed Service for Apache Spark and complete practical command-line exercises. Always attempt the lab yourself first and follow Google Cloud Skills Boost / Qwiklabs Terms of Service. This walkthrough is n

clear
CYAN='\033[1;36m'
GREEN='\033[1;32m'
YELLOW='\033[1;33m'
BLUE='\033[1;34m'
RESET='\033[0m'
BOLD='\033[1m'

echo -e "${CYAN}${BOLD}"
echo "   ____       _    _ _             __   ___             "
echo "  / __ \     | |  (_) |           / _| / _ \            "
echo " | |  | |_ __| |__  _| |_  ___   | |_ | | | |_ __ ___  "
echo " | |  | | '__| '_ \| | __| / _ \ |  _|| | | | '_ \ / __| "
echo " | |__| | |  | |_) | | |_ | (_) || |  | |_| | |_) \__ \ "
echo "  \____/|_|  |_.__/|_|\__| \___/ |_|   \___/| .__/|___/ "
echo "                                            | |         "
echo "                                            |_|         "
echo -e "${RESET}"

read -p "▶ Enter your assigned REGION (e.g., us-central1): " REGION

export PROJECT_ID=$(gcloud config get-value project)
export PROJECT_NUMBER=$(gcloud projects describe $PROJECT_ID --format='value(projectNumber)')

echo -e "${YELLOW}▶ Fixing Dataproc API state (Disabling then re-enabling)...${RESET}"
gcloud services disable dataproc.googleapis.com --force --quiet
gcloud services enable dataproc.googleapis.com --quiet

echo -e "${CYAN}▶ Waiting 30 seconds for the Dataproc Service Agent to propagate...${RESET}"
sleep 30

echo -e "${YELLOW}▶ Configuring IAM roles and Private Google Access...${RESET}"
gcloud projects add-iam-policy-binding $PROJECT_ID \
  --member=serviceAccount:[email protected] \
  --role=roles/storage.admin --quiet

gcloud projects add-iam-policy-binding $PROJECT_ID \
  --member=serviceAccount:[email protected] \
  --role=roles/dataproc.worker --quiet

gcloud compute networks subnets update default \
  --region=$REGION \
  --enable-private-ip-google-access --quiet

echo -e "${BLUE}▶ Task 1: Creating Dataproc cluster 'example-cluster' (This takes ~2 minutes)...${RESET}"
gcloud dataproc clusters create example-cluster \
  --region=$REGION \
  --worker-boot-disk-size=500 \
  --worker-machine-type=e2-standard-4 \
  --master-machine-type=e2-standard-4 \
  --quiet

echo -e "${YELLOW}▶ Task 2: Submitting SparkPi job...${RESET}"
gcloud dataproc jobs submit spark \
  --cluster=example-cluster \
  --region=$REGION \
  --class=org.apache.spark.examples.SparkPi \
  --jars=file:///usr/lib/spark/examples/jars/spark-examples.jar \
  -- 1000

echo -e "\n${GREEN}${BOLD}✓ Tasks 1 and 2 completed successfully! You may now click 'Check my progress'.${RESET}"

echo -e "${CYAN}▶ Optional: If your lab requires Task 3 (Modify number of workers), run this command manually:${RESET}"
echo "gcloud dataproc clusters update example-cluster --region=$REGION --num-workers=4"