Crawler to Create, Launch and Activate Your Website’s Cache

#!/bin/bash

# Usage: crawler.sh --url <sitemap-url> --threads <number-of-threads>

USER_AGENT="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"

# Function to process each URL and show the HTTP status code
process_url() {
    local url=$1
    local http_status=$(curl -o /dev/null -s -w "%{http_code}\n" -A "$USER_AGENT" "$url")
    echo "URL: $url - HTTP Code: $http_status"
}

# Function to process the sitemap
process_sitemap() {
    local sitemap_url=$1
    local threads=$2
    local temp_file=$(mktemp)

    # Download the sitemap
    curl -s -A "$USER_AGENT" "$sitemap_url" -o "$temp_file"

    # Extract URLs
    grep -oP '<loc>(.*?)</loc>' "$temp_file" | sed -e 's/<\/*loc>//g' | xargs -n 1 -P "$threads" -I {} bash -c "
        process_url() {
            local url=\$1
            local http_status=\$(curl -o /dev/null -s -w \"%{http_code}\n\" -A \"$USER_AGENT\" \"\$url\")
            echo \"URL: \$url - HTTP Code: \$http_status\"
        }
        process_url '{}'
    "

    # Clean up the temporary file
    rm "$temp_file"
}

# Parse arguments
while [[ "$#" -gt 0 ]]; do
    case $1 in
        --url) sitemap_url="$2"; shift ;;
        --threads) threads="$2"; shift ;;
        *) echo "Unknown option: $1" ;;
    esac
    shift
done

# Check that the required arguments have been given
if [[ -z "$sitemap_url" || -z "$threads" ]]; then
    echo "Missing arguments. Usage: crawler.sh --url <url> --threads <number>"
    exit 1
fi

# Call the main function
process_sitemap "$sitemap_url" "$threads"