Prepare Data for ML APIs on Google Cloud

Solution for Prepare Data for ML APIs on Google Cloud. 1 lab: GSP323. Fast copy-paste commands for Google Cloud.

GSP323 — Prepare Data for ML APIs on Google Cloud: Challenge Lab

Estimated time: 45 minutes

# 🚀 Prepare Data for ML APIs on Google Cloud: Challenge Lab > ⚠️ **Disclaimer:** This is an independent, community-made walkthrough created for educational purposes, hands-on practice, and Google Cloud certification preparation. This guide is designed to help learners understand data processing, machine learning APIs, and practical Google Cloud workflows involving Dataflow, Dataproc, Speech-to-Text, and Natural Language. Always attempt the lab yourself first and follow Google Cloud Skills Boos

# ==============================================================================
# ORBIT OF OPS - GSP323 (COMMAND 1 OF 3: SETUP)
# ==============================================================================
GREEN='\e[1;32m'
CYAN='\e[1;36m'
YELLOW='\e[1;33m'
MAGENTA='\e[1;35m'
RESET='\e[0m'
BOLD='\e[1m'

clear
echo -e "${CYAN}${BOLD}"
cat << "EOF"
  ____       _     _ _            __    ___            
 / __ \     | |   (_) |          / _|  / _ \           
| |  | |_ __| |__  _| |_   ___  | |_  | | | |_ __  ___ 
| |  | | '__| '_ \| | __| / _ \ |  _| | | | | '_ \/ __|
| |__| | |  | |_) | | |_ | (_) || |   | |_| | |_) \__ \
 \____/|_|  |_.__/|_|\__| \___/ |_|    \___/| .__/|___/
                                            | |        
                                            |_|        
EOF
echo -e "${RESET}"
echo -e "${MAGENTA}${BOLD}>>> ORBIT OF OPS: GSP323 INITIATED <<<${RESET}\n"

echo -e "${BOLD}${YELLOW}[Orbit of Ops] Auto-fetching Project, Zone, and Region...${RESET}"
export PROJECT_ID=$(gcloud config get-value project 2>/dev/null)
export ZONE=$(gcloud compute project-info describe --format="value(commonInstanceMetadata.items[google-compute-default-zone])" 2>/dev/null | tail -n 1)

if [[ -z "$ZONE" ]]; then
    read -p "$(echo -e "${BOLD}${CYAN}Please enter the lab Zone (e.g., us-east1-b): ${RESET}") " ZONE
    export ZONE
fi

export REGION=${ZONE%-*}
gcloud config set compute/zone $ZONE 2>/dev/null
gcloud config set compute/region $REGION 2>/dev/null

echo -e "✅ Project ID: ${GREEN}$PROJECT_ID${RESET}"
echo -e "✅ Region:     ${GREEN}$REGION${RESET}\n"

echo -e "${YELLOW}${BOLD}--- REQUIRED LAB INPUTS ---${RESET}"
read -p "$(echo -e "${CYAN}${BOLD}1. Enter BigQuery Dataset Name: ${RESET}") " DATASET
read -p "$(echo -e "${CYAN}${BOLD}2. Enter BigQuery Output Table Name: ${RESET}") " TABLE
read -p "$(echo -e "${MAGENTA}${BOLD}3. Enter Task 3 Cloud Speech Location (gs://...): ${RESET}") " TASK3_OUTPUT
read -p "$(echo -e "${MAGENTA}${BOLD}4. Enter Task 4 Cloud Natural Language Location (gs://...): ${RESET}") " TASK4_OUTPUT

export BUCKET="${PROJECT_ID}-marking"
export TEMP_LOCATION="gs://${BUCKET}/temp"
export BQ_TEMP="gs://${BUCKET}/bigquery_temp"

echo -e "\n${BLUE}${BOLD}[Orbit of Ops] Creating BigQuery Dataset and Storage Bucket...${RESET}"
bq mk --location=US $DATASET 2>/dev/null || true
gcloud storage buckets create gs://$BUCKET --location=$REGION --quiet || true

echo -e "\n${GREEN}${BOLD}🎉 COMMAND 1 COMPLETE! Proceed to Command 2.${RESET}"

# ==============================================================================
# ORBIT OF OPS - GSP323 (COMMAND 2 OF 3: DATA PIPELINES)
# ==============================================================================

echo -e "${YELLOW}${BOLD}[Orbit of Ops] Task 1: Initiating Dataflow Batch Job...${RESET}"
gcloud dataflow jobs run batch-job-task1 \
  --gcs-location gs://dataflow-templates-$REGION/latest/GCS_Text_to_BigQuery \
  --region $REGION \
  --worker-machine-type e2-standard-2 \
  --staging-location $TEMP_LOCATION \
  --parameters \
javascriptTextTransformFunctionName=transform,\
JSONPath=gs://spls/gsp323/lab.schema,\
javascriptTextTransformGcsPath=gs://spls/gsp323/lab.js,\
inputFilePattern=gs://spls/gsp323/lab.csv,\
outputTable=$PROJECT_ID:$DATASET.$TABLE,\
bigQueryLoadingTemporaryDirectory=$BQ_TEMP \
  --quiet

echo -e "\n${MAGENTA}${BOLD}[Orbit of Ops] Task 2: Provisioning Dataproc Cluster (Takes ~2 mins)...${RESET}"
gcloud dataproc clusters create cluster-task2 \
    --region=$REGION \
    --num-workers=2 \
    --master-machine-type=n2d-standard-2 \
    --master-boot-disk-type=pd-standard \
    --master-boot-disk-size=100 \
    --worker-machine-type=n2d-standard-2 \
    --worker-boot-disk-type=pd-standard \
    --worker-boot-disk-size=100 \
    --image-version=2.0-debian10 \
    --project=$PROJECT_ID \
    --quiet

export MASTER_NODE=$(gcloud compute instances list --filter="name ~ cluster-task2-m" --format="value(name)")
export MASTER_ZONE=$(gcloud compute instances list --filter="name ~ cluster-task2-m" --format="value(zone)")

echo -e "${CYAN}${BOLD}[Orbit of Ops] Seeding HDFS Data on Master Node...${RESET}"
gcloud compute ssh $MASTER_NODE --zone=$MASTER_ZONE --quiet --command="gcloud storage cp gs://spls/gsp323/data.txt . && hdfs dfs -put data.txt /data.txt"

echo -e "${BLUE}${BOLD}[Orbit of Ops] Submitting Apache Spark Job...${RESET}"
gcloud dataproc jobs submit spark \
    --cluster=cluster-task2 \
    --region=$REGION \
    --class=org.apache.spark.examples.SparkPageRank \
    --jars=file:///usr/lib/spark/examples/jars/spark-examples.jar \
    --max-failures-per-hour=1 \
    --quiet \
    -- /data.txt

echo -e "\n${GREEN}${BOLD}🎉 COMMAND 2 COMPLETE! Tasks 1 and 2 are processing in the background. Proceed to Command 3.${RESET}"

# ==============================================================================
# ORBIT OF OPS - GSP323 (COMMAND 3 OF 3: ML APIS)
# ==============================================================================

echo -e "${YELLOW}${BOLD}[Orbit of Ops] Task 3: Enabling APIs and Generating Key...${RESET}"
gcloud services enable apikeys.googleapis.com speech.googleapis.com --quiet
gcloud alpha services api-keys create --display-name="ml-api-key" --api-target=service=speech.googleapis.com --quiet

echo -e "${CYAN}${BOLD}[Orbit of Ops] Waiting 30 seconds for API Key propagation...${RESET}"
sleep 30

export KEY_NAME=$(gcloud alpha services api-keys list --format="value(name)" --filter="displayName=ml-api-key" --limit=1)
export API_KEY=$(gcloud alpha services api-keys get-key-string "$KEY_NAME" --format="value(keyString)")

cat > request.json <<EOF
{
  "config": {
    "encoding": "FLAC",
    "languageCode": "en-US"
  },
  "audio": {
    "uri": "gs://spls/gsp323/task3.flac"
  }
}
EOF

echo -e "${BLUE}${BOLD}[Orbit of Ops] Calling Speech-to-Text API...${RESET}"
curl -s -X POST -H "Content-Type: application/json" --data-binary @request.json "https://speech.googleapis.com/v1/speech:recognize?key=${API_KEY}" > result_task3.json

echo -e "${MAGENTA}${BOLD}[Orbit of Ops] Uploading Task 3 Result...${RESET}"
gcloud storage cp --content-type="application/json" result_task3.json $TASK3_OUTPUT --quiet

echo -e "\n${YELLOW}${BOLD}[Orbit of Ops] Task 4: Calling Natural Language API...${RESET}"
gcloud ml language analyze-entities --content="Old Norse texts portray Odin as one-eyed and long-bearded, frequently wielding a spear named Gungnir and wearing a cloak and a broad hat." > result_task4.json

echo -e "${CYAN}${BOLD}[Orbit of Ops] Uploading Task 4 Result...${RESET}"
gcloud storage cp --content-type="application/json" result_task4.json $TASK4_OUTPUT --quiet

echo -e "\n${GREEN}${BOLD}🎉 ALL AUTOMATION COMPLETE!${RESET}"
echo -e "${YELLOW}${BOLD}Important: Dataflow and Dataproc jobs from Command 2 might still be running. Monitor them in the console, wait for them to finish, and then click 'Check my progress' on all tasks!${RESET}"