47 lines
926 B
Bash
Executable File
47 lines
926 B
Bash
Executable File
#!/bin/bash
|
|
|
|
if [ $# -eq 0 ]; then
|
|
echo "Usage: $0 file_with_urls.txt"
|
|
exit 1
|
|
fi
|
|
|
|
URL_FILE="$1"
|
|
OUTPUT_DIR="output"
|
|
|
|
mkdir -p "$OUTPUT_DIR"
|
|
|
|
while IFS= read -r url; do
|
|
if [ -z "$url" ]; then
|
|
continue
|
|
fi
|
|
|
|
domain=$(echo "$url" | awk -F[/:] '{print $4}')
|
|
|
|
filename=$(basename "$url")
|
|
|
|
sub_dir="$OUTPUT_DIR/$domain"
|
|
full_path="${sub_dir}/${filename}"
|
|
|
|
mkdir -p "$sub_dir"
|
|
|
|
if [ ! -f "$full_path" ]; then
|
|
echo "Downloading $url..."
|
|
echo "Subdir $sub_dir"
|
|
|
|
curl -o "$full_path" "$url"
|
|
|
|
last_modified=$(curl -sI $url | grep -i Last-Modified | awk '{print substr($0, index($0,$2))}')
|
|
|
|
echo "Modified: $last_modified"
|
|
touch -d "$last_modified" "$full_path"
|
|
|
|
echo "---"
|
|
else
|
|
echo "Skipping $url; file already exists."
|
|
fi
|
|
|
|
sleep 2
|
|
done < "$URL_FILE"
|
|
|
|
echo "Download complete."
|