Automate Data Capture at Scale with Document AI
Solution for Automate Data Capture at Scale with Document AI. 1 lab: GSP367. Fast copy-paste commands for Google Cloud.
GSP367 — Automate Data Capture at Scale with Document AI: Challenge Lab
Estimated time: 30 minutes
# 🚀 Automate Data Capture at Scale with Document AI: Challenge Lab > ⚠️ **Disclaimer:** This is an independent, community-made walkthrough created for educational purposes, hands-on practice, and Google Cloud certification preparation. This guide is designed to help learners understand Document AI, event-driven data processing, Cloud Storage, BigQuery, Cloud Run Functions, Eventarc, and related Google Cloud services through hands-on exercises. Always attempt the lab yourself first and follow G
# ==============================================================================
# ORBIT OF OPS - GSP367 (COMMAND 1 OF 4: SETUP & CLONING - TASK 1 FIX)
# ==============================================================================
GREEN='\e[1;32m'
CYAN='\e[1;36m'
YELLOW='\e[1;33m'
BLUE='\e[1;34m'
MAGENTA='\e[1;35m'
RED='\e[1;31m'
RESET='\e[0m'
BOLD='\e[1m'
clear
echo -e "${CYAN}${BOLD}"
cat << "EOF"
____ _ _ _ __ ___
/ __ \ | | (_) | / _| / _ \
| | | |_ __| |__ _| |_ ___ | |_ | | | |_ __ ___
| | | | '__| '_ \| | __| / _ \ | _| | | | | '_ \/ __|
| |__| | | | |_) | | |_ | (_) || | | |_| | |_) \__ \
\____/|_| |_.__/|_|\__| \___/ |_| \___/| .__/|___/
| |
|_|
EOF
echo -e "${RESET}"
echo -e "${MAGENTA}${BOLD}>>> ORBIT OF OPS: GSP367 INITIATED <<<${RESET}\n"
echo -e "${BOLD}${YELLOW}[Orbit of Ops] Auto-fetching Project, Zone, and Region...${RESET}"
export PROJECT_ID=$(gcloud config get-value project 2>/dev/null)
if [[ -z "$PROJECT_ID" ]]; then export PROJECT_ID=$DEVSHELL_PROJECT_ID; fi
export PROJECT_NUMBER=$(gcloud projects describe $PROJECT_ID --format='value(projectNumber)' 2>/dev/null)
export ZONE=$(gcloud compute project-info describe \
--format="value(commonInstanceMetadata.items[google-compute-default-zone])" 2>/dev/null | tail -n 1)
if [[ -z "$ZONE" ]]; then
read -p "${BOLD}${CYAN}Please enter the lab Zone (e.g., us-east1-b): ${RESET}" ZONE
export ZONE
fi
export REGION=${ZONE%-*}
gcloud config set compute/zone $ZONE 2>/dev/null
gcloud config set compute/region $REGION 2>/dev/null
echo -e "✅ Project ID: ${GREEN}$PROJECT_ID${RESET}"
echo -e "✅ Zone: ${GREEN}$ZONE${RESET}"
echo -e "✅ Region: ${GREEN}$REGION${RESET}\n"
echo -e "${BLUE}${BOLD}[Orbit of Ops] Enabling Required APIs...${RESET}"
gcloud services enable documentai.googleapis.com cloudfunctions.googleapis.com cloudbuild.googleapis.com eventarc.googleapis.com run.googleapis.com --quiet
echo -e "${CYAN}${BOLD}[Orbit of Ops] Cloning Source Repository (Strict adherece for scoring script)...${RESET}"
cd ~
mkdir -p ./document-ai-challenge
gcloud storage cp -r gs://spls/gsp367/* ~/document-ai-challenge/ --quiet
echo -e "${YELLOW}${BOLD}[Orbit of Ops] Double-pinging Document AI API to force backend sync...${RESET}"
gcloud services enable documentai.googleapis.com --quiet
echo -e "\n${GREEN}${BOLD}🎉 COMMAND 1 COMPLETE! You can now click 'Check my progress' for Task 1.${RESET}"# ==============================================================================
# ORBIT OF OPS - GSP367 (COMMAND 2 OF 4: STORAGE & IAM)
# ==============================================================================
echo -e "${YELLOW}${BOLD}[Orbit of Ops] Creating Cloud Storage Buckets...${RESET}"
gcloud storage buckets create gs://${PROJECT_ID}-input-invoices --location=$REGION --uniform-bucket-level-access --quiet || true
gcloud storage buckets create gs://${PROJECT_ID}-output-invoices --location=$REGION --uniform-bucket-level-access --quiet || true
gcloud storage buckets create gs://${PROJECT_ID}-archived-invoices --location=$REGION --uniform-bucket-level-access --quiet || true
echo -e "${BLUE}${BOLD}[Orbit of Ops] Provisioning BigQuery Dataset and Table...${RESET}"
bq --location=US mk -d --description "Form Parser Results" ${PROJECT_ID}:invoice_parser_results 2>/dev/null || true
bq mk --table invoice_parser_results.doc_ai_extracted_entities ~/document-ai-challenge/scripts/table-schema/doc_ai_extracted_entities.json 2>/dev/null || true
echo -e "${MAGENTA}${BOLD}[Orbit of Ops] Instantiating Service Agents & Assigning IAM...${RESET}"
gcloud beta services identity create --service=cloudfunctions.googleapis.com --project=$PROJECT_ID 2>/dev/null || true
gcloud beta services identity create --service=eventarc.googleapis.com --project=$PROJECT_ID 2>/dev/null || true
export COMPUTE_SA="${PROJECT_NUMBER}[email protected]"
RAW_SA=$(gcloud storage service-agent --project=$PROJECT_ID 2>/dev/null | tr -d '[:space:]')
if [ -z "$RAW_SA" ]; then RAW_SA="service-${PROJECT_NUMBER}@gs-project-accounts.iam.gserviceaccount.com"; fi
gcloud projects add-iam-policy-binding $PROJECT_ID --member="serviceAccount:${COMPUTE_SA}" --role="roles/owner" --quiet >/dev/null
gcloud projects add-iam-policy-binding $PROJECT_ID --member="serviceAccount:${COMPUTE_SA}" --role="roles/eventarc.eventReceiver" --quiet >/dev/null
gcloud projects add-iam-policy-binding $PROJECT_ID --member="serviceAccount:${RAW_SA}" --role="roles/pubsub.publisher" --quiet >/dev/null
gcloud projects add-iam-policy-binding $PROJECT_ID --member="serviceAccount:${RAW_SA}" --role="roles/iam.serviceAccountTokenCreator" --quiet >/dev/null
echo -e "${YELLOW}${BOLD}Waiting 30 seconds for IAM policies to propagate...${RESET}"
sleep 30
echo -e "\n${GREEN}${BOLD}🎉 COMMAND 2 COMPLETE! You can now click 'Check my progress' for Task 3.${RESET}"# ==============================================================================
# ORBIT OF OPS - GSP367 (COMMAND 3 OF 4: DOCUMENT AI)
# ==============================================================================
read -p "$(echo -e ${MAGENTA}${BOLD}Enter the exact Processor Name from your lab instructions: ${RESET})" PROCESSOR_NAME
if [[ -z "$PROCESSOR_NAME" ]]; then PROCESSOR_NAME="processor name"; fi
echo -e "${CYAN}${BOLD}[Orbit of Ops] Creating Document AI Form Parser...${RESET}"
export ACCESS_TOKEN=$(gcloud auth application-default print-access-token)
curl -s -X POST -H "Authorization: Bearer $ACCESS_TOKEN" -H "Content-Type: application/json" \
-d "{\"display_name\": \"$PROCESSOR_NAME\", \"type\": \"FORM_PARSER_PROCESSOR\"}" \
"https://documentai.googleapis.com/v1/projects/$PROJECT_ID/locations/us/processors" > /dev/null
echo -e "${BLUE}${BOLD}[Orbit of Ops] Extracting Processor ID...${RESET}"
export PROCESSOR_ID=$(curl -s -X GET -H "Authorization: Bearer $ACCESS_TOKEN" "https://documentai.googleapis.com/v1/projects/$PROJECT_ID/locations/us/processors" | jq -r '.processors[] | select(.displayName=="'"$PROCESSOR_NAME"'") | .name' | awk -F'/' '{print $6}' | head -n 1)
if [[ -z "$PROCESSOR_ID" ]]; then
echo -e "${RED}${BOLD}Failed to auto-extract Processor ID. Please check the Document AI console.${RESET}"
read -p "Manually paste the Processor ID here: " PROCESSOR_ID
fi
echo -e "✅ Processor ID: ${GREEN}$PROCESSOR_ID${RESET}"
echo -e "\n${GREEN}${BOLD}🎉 COMMAND 3 COMPLETE! You can now click 'Check my progress' for Task 2.${RESET}"# ==============================================================================
# ORBIT OF OPS - GSP367 (COMMAND 4 OF 4: DEPLOY & TEST)
# ==============================================================================
echo -e "${MAGENTA}${BOLD}[Orbit of Ops] Deploying Cloud Run Function (Takes ~3-4 mins)...${RESET}"
cd ~/document-ai-challenge/scripts
export DEPLOY_CMD="gcloud functions deploy process-invoices \
--gen2 \
--region=${REGION} \
--entry-point=process_invoice \
--runtime=python313 \
--service-account=${COMPUTE_SA} \
--source=cloud-functions/process-invoices \
--timeout=400 \
--set-env-vars=\"PROJECT_ID=${PROJECT_ID},GCP_PROJECT=${PROJECT_ID},PROCESSOR_ID=${PROCESSOR_ID},PARSER_LOCATION=us,TIMEOUT=400\" \
--trigger-resource=gs://${PROJECT_ID}-input-invoices \
--trigger-event=google.storage.object.finalize \
--allow-unauthenticated \
--max-instances=1 \
--quiet"
eval $DEPLOY_CMD || { echo -e "${RED}Retrying deployment to allow IAM propagation...${RESET}"; sleep 45; eval $DEPLOY_CMD; }
echo -e "${YELLOW}${BOLD}[Orbit of Ops] Uploading test invoices to trigger pipeline...${RESET}"
gcloud storage cp ~/document-ai-challenge/invoices/* gs://${PROJECT_ID}-input-invoices/ --quiet
echo -e "\n${GREEN}${BOLD}🎉 AUTOMATION COMPLETE!${RESET}"
echo -e "${CYAN}${BOLD}The pipeline has been triggered. Please wait 1-2 minutes for the data to process into BigQuery, then click 'Check my progress' for Tasks 4 and 5!${RESET}"