#!/bin/bash # This script will scrape moonrank.app and save a json file of the # rarities for all NFTs of a collection (must be fully minted). ########## Configurable Variables ########## # Where to save the json files # # File location: (folder) # directory="~" # # # How long to wait for a response # # Curl timeout: (seconds) # ctimeout=5 # # # How long to sleep between web queries # # Delay: (seconds) # delay=2 # # # How many checks to do between sleeps # # Count: (number) # sequential=2 # # ############################################ # Colors are fun blk=$(tput setaf 0);red=$(tput setaf 1);grn=$(tput setaf 2) ylw=$(tput setaf 3);blu=$(tput setaf 4);cyn=$(tput setaf 6) wht=$(tput setaf 7);und=$(tput smul);bld=$(tput bold) # Uncoloring is important rst=$(tput sgr0) json="" max=0 cur=0 moonrank="https://moonrank.app/collection/" trap ctrl_c INT function ctrl_c() { tput cnorm echo "${rst}${red}CTRL+C Detected, exiting.${rst}" exit } function timedout() { echo -e "\n${red}No response received for ${bld}${1}${rst}${red}." echo -e "Please ensure you are able to access the internet. If this problem" echo -e "persists perhaps moonrank.app is unavailable. Consult your friends.${rst}\n" exit 1 } function verifyurl() { status=`curl -I -m ${ctimeout} "$1" 2>/dev/null | head -n1 | cut -d' ' -f2` [ "$status" == "" ] && timedout "${1}" [ $status -eq 200 ] && return 0 || return 1 } function getcollectioninfo() { local moonrankdata=`curl -m ${ctimeout} ${moonrank}${collection} 2>/dev/null` [ "${moonrankdata}" == "" ] && timedout "${1}" name=`echo "${moonrankdata}" | grep "font-medium text-white tracking-tight hyphens-auto" -a1 | tail -n1` count=`echo "${moonrankdata}" | egrep -o "[0-9,]+ pieces" | cut -d' ' -f1` verifyurl "${moonrank}${collection}/0" zeroindexed=`[ $? -eq 0 ] && echo yes || echo no` } # Sort through the HTML and save the rarity function getrarity() { local moonrankdata=`curl -m ${ctimeout} ${moonrank}${collection}/${cur} 2>/dev/null` [ "${moonrankdata}" == "" ] && timedout "${1}" rarity=`echo "${moonrankdata}" | egrep -o "text=\"rank\">\d+" | egrep -o "\d+"` nftname=`echo "${moonrankdata}" | egrep -o "aria\-current=\"page\">[a-zA-Z ]+#[0-9]+" | cut -d">" -f2` } # $1=key $2=value function appendjson() { [ "${#json}" -eq 0 ] && json="{\"$1\":$2" || json="${json}, \"$1\":$2" } function pad() { [ $1 -ne 0 ] && seq -s " " $1 | tr -d "[:digit:]" } function isodd(){ local num=$1 [ $num -eq 0 ] && return 0 [ $num -lt 0 ] && let num="num*-1" local a=0;local b=0 let a="num/2" let b="(num-1)/2" [ $a -eq $b ] && return 1 return 0 } function tableheader() { [ $didheader -eq 2 ] && return 1 didheader=0 local namelen=${#nftname} isodd $namelen local odd=$? if [[ $namelen -gt 7 ]]; then let prepost="namelen-7" let prepost="prepost/2" elif [[ $namelen -lt 7 ]]; then let prepost="7-namelen" let prepost="prepost/2" else prepost=0 fi for i in `seq 2`; do [ $didheader -eq 0 ] && echo -n " " pad $prepost [ $prepost -ne 0 ] && echo -n " " echo -n "NFT ID" [ $odd -eq 0 ] && let prepost="prepost+1" pad $prepost [ $prepost -eq 0 ] && echo -en "\t" echo -en "\tNFT Rarity Rank" [ $didheader -eq 0 ] && echo -en "\t\t" [ $didheader -eq 1 ] && echo let didheader="didheader+1" done tablecolumn=1 } function tablerow() { if [[ $tablecolumn -eq 1 ]]; then echo -en " ${nftname}\t ${rarity}" tablecolumn=2 else echo -e "\t\t ${nftname}\t ${rarity}" tablecolumn=1 fi } ####### START yn=n while [[ $yn == n ]]; do echo -en "\nPlease enter the moonrank collection ID: ${cyn}" read collection echo ${rst} collection=`echo $collection | tr [:blank:] _` echo -en "You entered ${bld}${collection}${rst}. Is this correct? [${grn}Y${rst}/${red}n${rst}]: ${cyn}" read yn echo -en "${rst}" done verifyurl ${moonrank}${collection} [ $? -eq 1 ] && echo -e "\n${red}${bld}${collection}${rst}${red} could not be found. Please ensure the name is correct and try again.${rst}\n" && exit 1 getcollectioninfo echo -e "\nFound the ${bld}${name}${rst} collection of ${bld}${count}${rst} NFTs." max=`echo ${count} | tr -d ,` [ $zeroindexed == "yes" ] && let max="max-1" || cur=1 echo -e "\n" didheader=0 thisseq=0 getrarity tableheader while [[ $cur -le $max ]]; do getrarity tablerow appendjson "${nftname}" ${rarity} let cur="cur+1" let thisseq="thisseq+1" [ $thisseq -eq $sequential ] && thisseq=0 && sleep $delay done json="${json}}" echo "$json" > ${directory}/${collection}-rarity.json echo -e "\nThe .json file for the ${name} NFT collection has been saved to ${directory}/${collection}-rarity.json\n"