diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 052f98e9..6bb2233d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,39 +13,32 @@ on: jobs: build: runs-on: ubuntu-latest - env: - BGPTOOLS_VERSION: 0.2.2 steps: - uses: actions/checkout@v3 - uses: dtolnay/rust-toolchain@stable - uses: cargo-bins/cargo-binstall@main - run: sudo apt-get install -y tree jq axel + - uses: extractions/setup-just@v3 - uses: actions/cache@v3 with: key: ${{ runner.os }}-build-dependencies path: | ~/.cargo/bin - - run: ./dependency.sh + - run: just dependency - uses: nick-fields/retry@v3 with: timeout_minutes: 60 max_attempts: 3 - command: ./generate.sh - - run: ./stat.sh - - run: ./guard.sh + command: just + - run: just guard - name: Checkout ip-lists branch uses: actions/checkout@v3 if: github.event_name == 'schedule' && github.ref == 'refs/heads/master' with: ref: ip-lists path: ip-lists - - run: ./upload.sh + - run: just upload if: github.event_name == 'schedule' && github.ref == 'refs/heads/master' - - name: Refresh CDN cache + - run: just refresh_jsdelivr ${{ github.repository }} if: github.event_name == 'schedule' && github.ref == 'refs/heads/master' - run: | - cd ip-lists - for file in *; do - curl -i "https://purge.jsdelivr.net/gh/${{ github.repository }}@ip-lists/${file}" - done diff --git a/common.sh b/common.sh deleted file mode 100755 index 464f271c..00000000 --- a/common.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash - -[[ $DEBUG == true ]] && set -x - -log_info(){ - >&2 echo "INFO>" $@ -} - -get_asn(){ - local CONF_FILE=$1 - unset PATTERN - unset COUNTRY - unset EXCLUDE - source $CONF_FILE - EXCLUDE=${EXCLUDE:-"^$"} - grep -P "${COUNTRY}\$" asnames.txt | - grep -Pi "$PATTERN" | - grep -vPi "$EXCLUDE" | - awk '{gsub(/AS/, ""); print $1 }' -} - -prepare_data_v4(){ - rm -f rib.gz - bgpkit-broker latest -c rrc00 --json | jq -c '.[] | select( .data_type | contains("rib")) | .url' | head -n 1 | xargs axel -q -o rib.gz - stat rib.gz - log_info "rib.gz ready for bgptools" -} -prepare_data_v6(){ - rm -f rib6.bz2 - bgpkit-broker latest -c route-views6 --json | jq -c '.[] | select( .data_type | contains("rib")) | .url' | head -n 1 | xargs axel -q -o rib6.bz2 - stat rib6.bz2 - log_info "rib6.bz2 ready for bgptools" -} -prepare_data(){ - curl -sSL https://bgp.potaroo.net/cidr/autnums.html | awk '-F[<>]' '{print $3,$5}' | grep '^AS' > asnames.txt & - prepare_data_v4 & - prepare_data_v6 & - wait_exit -} - -wait_exit(){ - local oldstate=$(set +o) - set +e - local s=0 - while [[ $s -ne 127 ]]; do - [[ $s -ne 0 ]] && exit $s - wait -n - s=$? - done - eval "$oldstate" - return 0 -} diff --git a/dependency.sh b/dependency.sh deleted file mode 100755 index 51a227aa..00000000 --- a/dependency.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -set -e - -bgptools --version | grep -F $BGPTOOLS_VERSION || \ - cargo install --version $BGPTOOLS_VERSION bgptools - -bgpkit-parser --version || \ - cargo binstall --secure --no-confirm bgpkit-parser@0.10.6 - -bgpkit-broker --version || \ - cargo binstall --secure --no-confirm bgpkit-broker@0.7.0 - -bgptools --version -bgpkit-parser --version -bgpkit-broker --version diff --git a/generate.sh b/generate.sh deleted file mode 100755 index aa537929..00000000 --- a/generate.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -set -e -source common.sh -[[ $SKIP_DATA_PREPARATION != true ]] && prepare_data -mkdir -p result -for file in operator/*.conf; do - operator=${file%.*} - operator=${operator##*/} - log_info "generating IP list of $operator ..." - get_asn $file - get_asn $file | xargs bgptools --mrt-file rib.gz | grep -Fv : | cat > result/${operator}.txt & - get_asn $file | xargs bgptools --mrt-file rib6.bz2 | grep -v '^::/0$' | grep -F : | cat > result/${operator}6.txt & -done - -wait_exit diff --git a/guard.sh b/guard.sh deleted file mode 100755 index 8980be82..00000000 --- a/guard.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -[[ $(wc -l result/china.txt) < 3000 ]] && exit 1 - -[[ $(wc -l result/china6.txt) < 1000 ]] && exit 2 - -exit 0 diff --git a/justfile b/justfile new file mode 100644 index 00000000..b9504ad5 --- /dev/null +++ b/justfile @@ -0,0 +1,215 @@ +set unstable +set script-interpreter := ['bash'] +bgptools_version := "0.2.2" + +default: prepare all stat + +[doc('Install or update bgp tooling dependencies')] +[script] +dependency: + set -euo pipefail + + if ! bgptools --version 2>/dev/null | grep -F "{{bgptools_version}}" >/dev/null; then + cargo install --version "{{bgptools_version}}" bgptools + fi + + if ! bgpkit-broker --version >/dev/null 2>&1; then + cargo binstall --secure --no-confirm bgpkit-broker@0.7.0 + fi + + bgptools --version + bgpkit-broker --version + +[doc('Download and normalize latest autnums list')] +[script] +prepare_autnums: + set -euo pipefail + + axel -q -o autnums.html https://bgp.potaroo.net/cidr/autnums.html + awk -F'[<>]' '{print $3,$5}' autnums.html | grep '^AS' > asnames.txt + rm -f autnums.html + echo "INFO> asnames.txt updated ($(wc -l < asnames.txt) entries)" >&2 + +[doc('Download the latest IPv4 RIB snapshot')] +[script] +prepare_rib_v4: + set -euo pipefail + + url="$(bgpkit-broker latest -c rrc00 --json \ + | jq -r '.[] | select(.data_type | contains("rib")) | .url' \ + | head -n 1)" + + if [[ -z "${url}" ]]; then + echo "Unable to determine IPv4 RIB download url" >&2 + exit 1 + fi + + rm -f rib.gz + axel -q -o rib.gz "${url}" + stat rib.gz + echo "INFO> rib.gz ready for bgptools" >&2 + +[doc('Download the latest IPv6 RIB snapshot')] +[script] +prepare_rib_v6: + set -euo pipefail + + url="$(bgpkit-broker latest -c route-views6 --json \ + | jq -r '.[] | select(.data_type | contains("rib")) | .url' \ + | head -n 1)" + + if [[ -z "${url}" ]]; then + echo "Unable to determine IPv6 RIB download url" >&2 + exit 1 + fi + + rm -f rib6.bz2 + axel -q -o rib6.bz2 "${url}" + stat rib6.bz2 + echo "INFO> rib6.bz2 ready for bgptools" >&2 + +[doc('Prepare data for generation')] +[parallel] +prepare: prepare_autnums prepare_rib_v4 prepare_rib_v6 + +[doc('Print ASN list for OPERATOR based on operator/*.conf')] +[script] +get_asn operator: + set -euo pipefail + + config="operator/{{operator}}.conf" + if [[ ! -f "${config}" ]]; then + echo "Unknown operator: {{operator}}" >&2 + exit 1 + fi + + if [[ ! -s asnames.txt ]]; then + echo "Missing asnames.txt. Run 'just prepare_autnums' first." >&2 + exit 1 + fi + + # shellcheck disable=SC1090 + source "${config}" + : "${COUNTRY:?COUNTRY must be set in ${config}}" + EXCLUDE="${EXCLUDE:-^$}" + PATTERN="${PATTERN:-}" + + grep -P "${COUNTRY}\$" asnames.txt \ + | grep -Pi "${PATTERN}" \ + | grep -vPi "${EXCLUDE}" \ + | awk '{gsub(/AS/, ""); print $1 }' + +[doc('Generate IP lists for a single operator')] +[parallel] +gen operator: (gen4 operator) (gen6 operator) + +[script] +gen4 operator: + set -euo pipefail + + mkdir -p result + + echo "INFO> generating IPv4 prefixes for {{operator}}" >&2 + just get_asn "{{operator}}" \ + | tee >(awk 'END { if (NR == 0) exit 1 }') \ + | xargs bgptools --mrt-file rib.gz \ + | grep -Fv ':' \ + > "result/{{operator}}.txt" + echo "INFO> {{operator}}.txt generated ($(wc -l < result/{{operator}}.txt) entries)" >&2 + +[script] +gen6 operator: + set -euo pipefail + + mkdir -p result + + echo "INFO> generating IPv6 prefixes for {{operator}}" >&2 + just get_asn "{{operator}}" \ + | tee >(awk 'END { if (NR == 0) exit 1 }') \ + | xargs bgptools --mrt-file rib6.bz2 \ + | grep -v '^::/0$' \ + | grep -F ':' \ + > "result/{{operator}}6.txt" || true # ignore empty output, since drpeng has no IPv6 prefixes + echo "INFO> {{operator}}6.txt generated ($(wc -l < result/{{operator}}6.txt) entries)" >&2 + +[doc('Generate IP lists for all operators sequentially')] +[parallel] +all: (gen "china") (gen "cernet") (gen "chinanet") (gen "cmcc") (gen "unicom") (gen "cstnet") (gen "drpeng") (gen "googlecn") + +[script] +guard: + set -euo pipefail + + if [[ $(wc -l < result/china.txt) -lt 3000 ]]; then + echo "china.txt too small" >&2 + exit 1 + fi + + if [[ $(wc -l < result/china6.txt) -lt 1000 ]]; then + echo "china6.txt too small" >&2 + exit 2 + fi + + echo "INFO> guard checks passed" >&2 + +[doc('Summarize total IPv4/IPv6 address space per operator')] +[script] +stat: + set -euo pipefail + cd result + + for file in *.txt; do + name="${file%.*}" + echo "${name}" + if [[ "${file}" == *6.txt ]]; then + base=48 + else + base=32 + fi + + sum=0 + while IFS=/ read -r _ mask; do + if [[ -z "${mask}" ]]; then + continue + fi + if (( mask <= base )); then + ((s=base-mask)) + ((sum+=1< index.html + git config user.name "GitHub Actions" + git config user.email noreply@github.com + git add . + git commit -m "update $(date +%Y-%m-%d)" + git push -q + +[doc('Refresh CDN cache for all files in ip-lists directory')] +[script] +refresh_jsdelivr repository: + set -euo pipefail + + if [[ ! -d ip-lists ]]; then + echo "ip-lists directory not found" >&2 + exit 1 + fi + + cd ip-lists + for file in *; do + if [[ -f "${file}" ]]; then + echo "INFO> purging CDN cache for ${file}" >&2 + curl -i "https://purge.jsdelivr.net/gh/{{repository}}@ip-lists/${file}" + fi + done + diff --git a/stat.sh b/stat.sh deleted file mode 100755 index 48cfe51e..00000000 --- a/stat.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -source common.sh -cd result -for file in *.txt; do - echo ${file%.*} - if [[ $file == *6.txt ]]; then - #statistics IPv6 /48 blocks - base=48 - else - base=32 - fi - cat $file | - awk -F\/ '{print $2}' | - ( - sum=0 - while read n; do - if [[ -n $n ]] && [[ $n -le $base ]]; then - ((s=base-n)) - ((sum+=1< index.html -git config user.name "GitHub Actions" -git config user.email noreply@github.com -git add . -git commit -m "update $(date +%Y-%m-%d)" -git push -q