mirror of
https://github.com/gaoyifan/china-operator-ip.git
synced 2025-12-16 06:13:15 +08:00
Refactor project with justfile
This commit is contained in:
Yifan Gao
19
.github/workflows/build.yml
vendored
19
.github/workflows/build.yml
vendored
@@ -13,39 +13,32 @@ on:
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
BGPTOOLS_VERSION: 0.2.2
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: cargo-bins/cargo-binstall@main
|
||||
- run: sudo apt-get install -y tree jq axel
|
||||
- uses: extractions/setup-just@v3
|
||||
- uses: actions/cache@v3
|
||||
with:
|
||||
key: ${{ runner.os }}-build-dependencies
|
||||
path: |
|
||||
~/.cargo/bin
|
||||
- run: ./dependency.sh
|
||||
- run: just dependency
|
||||
- uses: nick-fields/retry@v3
|
||||
with:
|
||||
timeout_minutes: 60
|
||||
max_attempts: 3
|
||||
command: ./generate.sh
|
||||
- run: ./stat.sh
|
||||
- run: ./guard.sh
|
||||
command: just
|
||||
- run: just guard
|
||||
- name: Checkout ip-lists branch
|
||||
uses: actions/checkout@v3
|
||||
if: github.event_name == 'schedule' && github.ref == 'refs/heads/master'
|
||||
with:
|
||||
ref: ip-lists
|
||||
path: ip-lists
|
||||
- run: ./upload.sh
|
||||
- run: just upload
|
||||
if: github.event_name == 'schedule' && github.ref == 'refs/heads/master'
|
||||
- name: Refresh CDN cache
|
||||
- run: just refresh_jsdelivr ${{ github.repository }}
|
||||
if: github.event_name == 'schedule' && github.ref == 'refs/heads/master'
|
||||
run: |
|
||||
cd ip-lists
|
||||
for file in *; do
|
||||
curl -i "https://purge.jsdelivr.net/gh/${{ github.repository }}@ip-lists/${file}"
|
||||
done
|
||||
|
||||
|
||||
52
common.sh
52
common.sh
@@ -1,52 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
[[ $DEBUG == true ]] && set -x
|
||||
|
||||
log_info(){
|
||||
>&2 echo "INFO>" $@
|
||||
}
|
||||
|
||||
get_asn(){
|
||||
local CONF_FILE=$1
|
||||
unset PATTERN
|
||||
unset COUNTRY
|
||||
unset EXCLUDE
|
||||
source $CONF_FILE
|
||||
EXCLUDE=${EXCLUDE:-"^$"}
|
||||
grep -P "${COUNTRY}\$" asnames.txt |
|
||||
grep -Pi "$PATTERN" |
|
||||
grep -vPi "$EXCLUDE" |
|
||||
awk '{gsub(/AS/, ""); print $1 }'
|
||||
}
|
||||
|
||||
prepare_data_v4(){
|
||||
rm -f rib.gz
|
||||
bgpkit-broker latest -c rrc00 --json | jq -c '.[] | select( .data_type | contains("rib")) | .url' | head -n 1 | xargs axel -q -o rib.gz
|
||||
stat rib.gz
|
||||
log_info "rib.gz ready for bgptools"
|
||||
}
|
||||
prepare_data_v6(){
|
||||
rm -f rib6.bz2
|
||||
bgpkit-broker latest -c route-views6 --json | jq -c '.[] | select( .data_type | contains("rib")) | .url' | head -n 1 | xargs axel -q -o rib6.bz2
|
||||
stat rib6.bz2
|
||||
log_info "rib6.bz2 ready for bgptools"
|
||||
}
|
||||
prepare_data(){
|
||||
curl -sSL https://bgp.potaroo.net/cidr/autnums.html | awk '-F[<>]' '{print $3,$5}' | grep '^AS' > asnames.txt &
|
||||
prepare_data_v4 &
|
||||
prepare_data_v6 &
|
||||
wait_exit
|
||||
}
|
||||
|
||||
wait_exit(){
|
||||
local oldstate=$(set +o)
|
||||
set +e
|
||||
local s=0
|
||||
while [[ $s -ne 127 ]]; do
|
||||
[[ $s -ne 0 ]] && exit $s
|
||||
wait -n
|
||||
s=$?
|
||||
done
|
||||
eval "$oldstate"
|
||||
return 0
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
bgptools --version | grep -F $BGPTOOLS_VERSION || \
|
||||
cargo install --version $BGPTOOLS_VERSION bgptools
|
||||
|
||||
bgpkit-parser --version || \
|
||||
cargo binstall --secure --no-confirm bgpkit-parser@0.10.6
|
||||
|
||||
bgpkit-broker --version || \
|
||||
cargo binstall --secure --no-confirm bgpkit-broker@0.7.0
|
||||
|
||||
bgptools --version
|
||||
bgpkit-parser --version
|
||||
bgpkit-broker --version
|
||||
16
generate.sh
16
generate.sh
@@ -1,16 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
source common.sh
|
||||
[[ $SKIP_DATA_PREPARATION != true ]] && prepare_data
|
||||
mkdir -p result
|
||||
for file in operator/*.conf; do
|
||||
operator=${file%.*}
|
||||
operator=${operator##*/}
|
||||
log_info "generating IP list of $operator ..."
|
||||
get_asn $file
|
||||
get_asn $file | xargs bgptools --mrt-file rib.gz | grep -Fv : | cat > result/${operator}.txt &
|
||||
get_asn $file | xargs bgptools --mrt-file rib6.bz2 | grep -v '^::/0$' | grep -F : | cat > result/${operator}6.txt &
|
||||
done
|
||||
|
||||
wait_exit
|
||||
7
guard.sh
7
guard.sh
@@ -1,7 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
[[ $(wc -l result/china.txt) < 3000 ]] && exit 1
|
||||
|
||||
[[ $(wc -l result/china6.txt) < 1000 ]] && exit 2
|
||||
|
||||
exit 0
|
||||
215
justfile
Normal file
215
justfile
Normal file
@@ -0,0 +1,215 @@
|
||||
set unstable
|
||||
set script-interpreter := ['bash']
|
||||
bgptools_version := "0.2.2"
|
||||
|
||||
default: prepare all stat
|
||||
|
||||
[doc('Install or update bgp tooling dependencies')]
|
||||
[script]
|
||||
dependency:
|
||||
set -euo pipefail
|
||||
|
||||
if ! bgptools --version 2>/dev/null | grep -F "{{bgptools_version}}" >/dev/null; then
|
||||
cargo install --version "{{bgptools_version}}" bgptools
|
||||
fi
|
||||
|
||||
if ! bgpkit-broker --version >/dev/null 2>&1; then
|
||||
cargo binstall --secure --no-confirm bgpkit-broker@0.7.0
|
||||
fi
|
||||
|
||||
bgptools --version
|
||||
bgpkit-broker --version
|
||||
|
||||
[doc('Download and normalize latest autnums list')]
|
||||
[script]
|
||||
prepare_autnums:
|
||||
set -euo pipefail
|
||||
|
||||
axel -q -o autnums.html https://bgp.potaroo.net/cidr/autnums.html
|
||||
awk -F'[<>]' '{print $3,$5}' autnums.html | grep '^AS' > asnames.txt
|
||||
rm -f autnums.html
|
||||
echo "INFO> asnames.txt updated ($(wc -l < asnames.txt) entries)" >&2
|
||||
|
||||
[doc('Download the latest IPv4 RIB snapshot')]
|
||||
[script]
|
||||
prepare_rib_v4:
|
||||
set -euo pipefail
|
||||
|
||||
url="$(bgpkit-broker latest -c rrc00 --json \
|
||||
| jq -r '.[] | select(.data_type | contains("rib")) | .url' \
|
||||
| head -n 1)"
|
||||
|
||||
if [[ -z "${url}" ]]; then
|
||||
echo "Unable to determine IPv4 RIB download url" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
rm -f rib.gz
|
||||
axel -q -o rib.gz "${url}"
|
||||
stat rib.gz
|
||||
echo "INFO> rib.gz ready for bgptools" >&2
|
||||
|
||||
[doc('Download the latest IPv6 RIB snapshot')]
|
||||
[script]
|
||||
prepare_rib_v6:
|
||||
set -euo pipefail
|
||||
|
||||
url="$(bgpkit-broker latest -c route-views6 --json \
|
||||
| jq -r '.[] | select(.data_type | contains("rib")) | .url' \
|
||||
| head -n 1)"
|
||||
|
||||
if [[ -z "${url}" ]]; then
|
||||
echo "Unable to determine IPv6 RIB download url" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
rm -f rib6.bz2
|
||||
axel -q -o rib6.bz2 "${url}"
|
||||
stat rib6.bz2
|
||||
echo "INFO> rib6.bz2 ready for bgptools" >&2
|
||||
|
||||
[doc('Prepare data for generation')]
|
||||
[parallel]
|
||||
prepare: prepare_autnums prepare_rib_v4 prepare_rib_v6
|
||||
|
||||
[doc('Print ASN list for OPERATOR based on operator/*.conf')]
|
||||
[script]
|
||||
get_asn operator:
|
||||
set -euo pipefail
|
||||
|
||||
config="operator/{{operator}}.conf"
|
||||
if [[ ! -f "${config}" ]]; then
|
||||
echo "Unknown operator: {{operator}}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -s asnames.txt ]]; then
|
||||
echo "Missing asnames.txt. Run 'just prepare_autnums' first." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC1090
|
||||
source "${config}"
|
||||
: "${COUNTRY:?COUNTRY must be set in ${config}}"
|
||||
EXCLUDE="${EXCLUDE:-^$}"
|
||||
PATTERN="${PATTERN:-}"
|
||||
|
||||
grep -P "${COUNTRY}\$" asnames.txt \
|
||||
| grep -Pi "${PATTERN}" \
|
||||
| grep -vPi "${EXCLUDE}" \
|
||||
| awk '{gsub(/AS/, ""); print $1 }'
|
||||
|
||||
[doc('Generate IP lists for a single operator')]
|
||||
[parallel]
|
||||
gen operator: (gen4 operator) (gen6 operator)
|
||||
|
||||
[script]
|
||||
gen4 operator:
|
||||
set -euo pipefail
|
||||
|
||||
mkdir -p result
|
||||
|
||||
echo "INFO> generating IPv4 prefixes for {{operator}}" >&2
|
||||
just get_asn "{{operator}}" \
|
||||
| tee >(awk 'END { if (NR == 0) exit 1 }') \
|
||||
| xargs bgptools --mrt-file rib.gz \
|
||||
| grep -Fv ':' \
|
||||
> "result/{{operator}}.txt"
|
||||
echo "INFO> {{operator}}.txt generated ($(wc -l < result/{{operator}}.txt) entries)" >&2
|
||||
|
||||
[script]
|
||||
gen6 operator:
|
||||
set -euo pipefail
|
||||
|
||||
mkdir -p result
|
||||
|
||||
echo "INFO> generating IPv6 prefixes for {{operator}}" >&2
|
||||
just get_asn "{{operator}}" \
|
||||
| tee >(awk 'END { if (NR == 0) exit 1 }') \
|
||||
| xargs bgptools --mrt-file rib6.bz2 \
|
||||
| grep -v '^::/0$' \
|
||||
| grep -F ':' \
|
||||
> "result/{{operator}}6.txt" || true # ignore empty output, since drpeng has no IPv6 prefixes
|
||||
echo "INFO> {{operator}}6.txt generated ($(wc -l < result/{{operator}}6.txt) entries)" >&2
|
||||
|
||||
[doc('Generate IP lists for all operators sequentially')]
|
||||
[parallel]
|
||||
all: (gen "china") (gen "cernet") (gen "chinanet") (gen "cmcc") (gen "unicom") (gen "cstnet") (gen "drpeng") (gen "googlecn")
|
||||
|
||||
[script]
|
||||
guard:
|
||||
set -euo pipefail
|
||||
|
||||
if [[ $(wc -l < result/china.txt) -lt 3000 ]]; then
|
||||
echo "china.txt too small" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ $(wc -l < result/china6.txt) -lt 1000 ]]; then
|
||||
echo "china6.txt too small" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
echo "INFO> guard checks passed" >&2
|
||||
|
||||
[doc('Summarize total IPv4/IPv6 address space per operator')]
|
||||
[script]
|
||||
stat:
|
||||
set -euo pipefail
|
||||
cd result
|
||||
|
||||
for file in *.txt; do
|
||||
name="${file%.*}"
|
||||
echo "${name}"
|
||||
if [[ "${file}" == *6.txt ]]; then
|
||||
base=48
|
||||
else
|
||||
base=32
|
||||
fi
|
||||
|
||||
sum=0
|
||||
while IFS=/ read -r _ mask; do
|
||||
if [[ -z "${mask}" ]]; then
|
||||
continue
|
||||
fi
|
||||
if (( mask <= base )); then
|
||||
((s=base-mask))
|
||||
((sum+=1<<s))
|
||||
fi
|
||||
done < "${file}"
|
||||
echo "${sum}"
|
||||
echo
|
||||
done | tee stat
|
||||
|
||||
[doc('Publish generated results into the ip-lists branch')]
|
||||
[script]
|
||||
upload: guard
|
||||
set -euo pipefail
|
||||
rm -f ip-lists/*.txt
|
||||
mv result/* ip-lists
|
||||
cd ip-lists
|
||||
tree -H . -P "*.txt|stat" -T "China Operator IP - prebuild results" > index.html
|
||||
git config user.name "GitHub Actions"
|
||||
git config user.email noreply@github.com
|
||||
git add .
|
||||
git commit -m "update $(date +%Y-%m-%d)"
|
||||
git push -q
|
||||
|
||||
[doc('Refresh CDN cache for all files in ip-lists directory')]
|
||||
[script]
|
||||
refresh_jsdelivr repository:
|
||||
set -euo pipefail
|
||||
|
||||
if [[ ! -d ip-lists ]]; then
|
||||
echo "ip-lists directory not found" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cd ip-lists
|
||||
for file in *; do
|
||||
if [[ -f "${file}" ]]; then
|
||||
echo "INFO> purging CDN cache for ${file}" >&2
|
||||
curl -i "https://purge.jsdelivr.net/gh/{{repository}}@ip-lists/${file}"
|
||||
fi
|
||||
done
|
||||
|
||||
26
stat.sh
26
stat.sh
@@ -1,26 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
source common.sh
|
||||
cd result
|
||||
for file in *.txt; do
|
||||
echo ${file%.*}
|
||||
if [[ $file == *6.txt ]]; then
|
||||
#statistics IPv6 /48 blocks
|
||||
base=48
|
||||
else
|
||||
base=32
|
||||
fi
|
||||
cat $file |
|
||||
awk -F\/ '{print $2}' |
|
||||
(
|
||||
sum=0
|
||||
while read n; do
|
||||
if [[ -n $n ]] && [[ $n -le $base ]]; then
|
||||
((s=base-n))
|
||||
((sum+=1<<s))
|
||||
fi
|
||||
done
|
||||
echo $sum
|
||||
)
|
||||
echo
|
||||
done | tee stat
|
||||
11
upload.sh
11
upload.sh
@@ -1,11 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
rm ip-lists/*.txt
|
||||
mv result/* ip-lists
|
||||
cd ip-lists
|
||||
tree -H . -P "*.txt|stat" -T "China Operator IP - prebuild results" > index.html
|
||||
git config user.name "GitHub Actions"
|
||||
git config user.email noreply@github.com
|
||||
git add .
|
||||
git commit -m "update $(date +%Y-%m-%d)"
|
||||
git push -q
|
||||
Reference in New Issue
Block a user