Merge category-*@cn to cn

.github/release-branch.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e -o pipefail
+
+mkdir -p release
+cd release
+git init
+git config --local user.email "github-action@users.noreply.github.com"
+git config --local user.name "GitHub Action"
+git remote add origin https://github-action:$GITHUB_TOKEN@github.com/SagerNet/sing-geosite.git
+git branch -M release
+cp ../*.db ../*.sha256sum .
+git add .
+git commit -m "Update release"
+git push -f origin release

.github/workflows/release.yaml

@@ -33,6 +33,12 @@ jobs:
         if: steps.build.outputs.skip != 'true'
         run: |
           sha256sum geosite.db > geosite.db.sha256sum
+          sha256sum geosite-cn.db > geosite-cn.db.sha256sum
+      - name: Release release branch
+        if: steps.build.outputs.skip != 'true'
+        run: .github/release-branch.sh
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - uses: dev-drprasad/delete-older-releases@v0.3.2
         if: steps.build.outputs.skip != 'true'
         with:
@@ -46,4 +52,6 @@ jobs:
           tag_name: ${{ steps.build.outputs.tag }}
           files: |
             geosite.db
-            sha256sum
+            geosite.db.sha256sum
+            geosite-cn.db
+            geosite-cn.db.sha256sum

go.mod

@@ -4,10 +4,10 @@ go 1.18
 
 require (
 	github.com/google/go-github/v45 v45.2.0
-	github.com/sagernet/sing v0.2.18-0.20231129075305-eb56a60214be
-	github.com/sagernet/sing-box v1.6.8-0.20231129123339-5a56487cf544
-	github.com/v2fly/v2ray-core/v5 v5.12.1
-	google.golang.org/protobuf v1.31.0
+	github.com/sagernet/sing v0.3.0-rc.7
+	github.com/sagernet/sing-box v1.8.0-rc.8
+	github.com/v2fly/v2ray-core/v5 v5.13.0
+	google.golang.org/protobuf v1.32.0
 )
 
 require (
@@ -16,11 +16,11 @@ require (
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/logrusorgru/aurora v2.0.3+incompatible // indirect
 	github.com/miekg/dns v1.1.57 // indirect
-	github.com/sagernet/sing-dns v0.1.11 // indirect
-	go4.org/netipx v0.0.0-20230824141953-6213f710f925 // indirect
-	golang.org/x/crypto v0.15.0 // indirect
+	github.com/sagernet/sing-dns v0.1.12 // indirect
+	go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect
+	golang.org/x/crypto v0.17.0 // indirect
 	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/net v0.18.0 // indirect
-	golang.org/x/sys v0.14.0 // indirect
-	golang.org/x/tools v0.15.0 // indirect
+	golang.org/x/net v0.19.0 // indirect
+	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/tools v0.16.0 // indirect
 )

go.sum

@@ -18,36 +18,36 @@ github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM=
 github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/sagernet/sing v0.2.18-0.20231129075305-eb56a60214be h1:FigAM9kq7RRXmHvgn8w2a8tqCY5CMV5GIk0id84dI0o=
-github.com/sagernet/sing v0.2.18-0.20231129075305-eb56a60214be/go.mod h1:OL6k2F0vHmEzXz2KW19qQzu172FDgSbUSODylighuVo=
-github.com/sagernet/sing-box v1.6.8-0.20231129123339-5a56487cf544 h1:kHe9kQpKMEmGJbTdDntXA0rl1BdQWQZtkDaxmZkAPC4=
-github.com/sagernet/sing-box v1.6.8-0.20231129123339-5a56487cf544/go.mod h1:C5Gcyr9BsHs1Iq7eAY53hAlh/j+5fjJVsLA18z/7ZL4=
-github.com/sagernet/sing-dns v0.1.11 h1:PPrMCVVrAeR3f5X23I+cmvacXJ+kzuyAsBiWyUKhGSE=
-github.com/sagernet/sing-dns v0.1.11/go.mod h1:zJ/YjnYB61SYE+ubMcMqVdpaSvsyQ2iShQGO3vuLvvE=
+github.com/sagernet/sing v0.3.0-rc.7 h1:FmnzFRYC6usVgWf112cUxiexwvL+iAurKmCL4Axa9+A=
+github.com/sagernet/sing v0.3.0-rc.7/go.mod h1:9pfuAH6mZfgnz/YjP6xu5sxx882rfyjpcrTdUpd6w3g=
+github.com/sagernet/sing-box v1.8.0-rc.8 h1:g51Iz9UueydCqPCCeIJk9D5uCa6PpGe4FcRc0vuH1m8=
+github.com/sagernet/sing-box v1.8.0-rc.8/go.mod h1:Rq9ZuuLKfd81AxUIHeD6R+FABCf9hlJqEQ0VOrtawBg=
+github.com/sagernet/sing-dns v0.1.12 h1:1HqZ+ln+Rezx/aJMStaS0d7oPeX2EobSV1NT537kyj4=
+github.com/sagernet/sing-dns v0.1.12/go.mod h1:rx/DTOisneQpCgNQ4jbFU/JNEtnz0lYcHXenlVzpjEU=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/v2fly/v2ray-core/v5 v5.12.1 h1:y7xdWEjQcjcNRjBbvFqWx9fdM/uMAp5kvy7xg45GQWk=
-github.com/v2fly/v2ray-core/v5 v5.12.1/go.mod h1:97vMnLdw9DdKPR+ZmHL1Qd7By5jMzE5vW5936et9nI0=
-go4.org/netipx v0.0.0-20230824141953-6213f710f925 h1:eeQDDVKFkx0g4Hyy8pHgmZaK0EqB4SD6rvKbUdN3ziQ=
-go4.org/netipx v0.0.0-20230824141953-6213f710f925/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y=
-golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA=
-golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g=
+github.com/v2fly/v2ray-core/v5 v5.13.0 h1:BDJfi3Ftx1NpQlZZPpeWJe3RDqRNyIVBs+YGG4RRMDU=
+github.com/v2fly/v2ray-core/v5 v5.13.0/go.mod h1:Bc3gmQWLr8UR7xBSCYI9FbfKuVvqA9lbkeBTWNRRAS4=
+go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M=
+go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y=
+golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
+golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
 golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
 golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg=
-golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ=
+golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
+golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
 golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
 golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
-golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/tools v0.15.0 h1:zdAyfUGbYmuVokhzVmghFl2ZJh5QhcfebBgmVPFYA+8=
-golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM=
+golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
-google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
+google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

main.go

@@ -8,17 +8,18 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 
 	"github.com/sagernet/sing-box/common/geosite"
 	"github.com/sagernet/sing-box/common/srs"
 	C "github.com/sagernet/sing-box/constant"
+	"github.com/sagernet/sing-box/log"
 	"github.com/sagernet/sing-box/option"
 	"github.com/sagernet/sing/common"
 	E "github.com/sagernet/sing/common/exceptions"
 
 	"github.com/google/go-github/v45/github"
-	"github.com/sagernet/sing-box/log"
 	"github.com/v2fly/v2ray-core/v5/app/router/routercommon"
 	"google.golang.org/protobuf/proto"
 )
@@ -169,12 +170,115 @@ func parse(vGeositeData []byte) (map[string][]geosite.Item, error) {
 	return domainMap, nil
 }
 
-func generate(release *github.RepositoryRelease, output string, ruleSetOutput string) error {
-	outputFile, err := os.Create(output)
-	if err != nil {
-		return err
+type filteredCodePair struct {
+	code    string
+	badCode string
+}
+
+func filterTags(data map[string][]geosite.Item) {
+	var codeList []string
+	for code := range data {
+		codeList = append(codeList, code)
 	}
-	defer outputFile.Close()
+	var badCodeList []filteredCodePair
+	var filteredCodeMap []string
+	var mergedCodeMap []string
+	for _, code := range codeList {
+		codeParts := strings.Split(code, "@")
+		if len(codeParts) != 2 {
+			continue
+		}
+		leftParts := strings.Split(codeParts[0], "-")
+		var lastName string
+		if len(leftParts) > 1 {
+			lastName = leftParts[len(leftParts)-1]
+		}
+		if lastName == "" {
+			lastName = codeParts[0]
+		}
+		if lastName == codeParts[1] {
+			delete(data, code)
+			filteredCodeMap = append(filteredCodeMap, code)
+			continue
+		}
+		if "!"+lastName == codeParts[1] {
+			badCodeList = append(badCodeList, filteredCodePair{
+				code:    codeParts[0],
+				badCode: code,
+			})
+		} else if lastName == "!"+codeParts[1] {
+			badCodeList = append(badCodeList, filteredCodePair{
+				code:    codeParts[0],
+				badCode: code,
+			})
+		}
+	}
+	for _, it := range badCodeList {
+		badList := data[it.badCode]
+		if badList == nil {
+			panic("bad list not found: " + it.badCode)
+		}
+		delete(data, it.badCode)
+		newMap := make(map[geosite.Item]bool)
+		for _, item := range data[it.code] {
+			newMap[item] = true
+		}
+		for _, item := range badList {
+			delete(newMap, item)
+		}
+		newList := make([]geosite.Item, 0, len(newMap))
+		for item := range newMap {
+			newList = append(newList, item)
+		}
+		data[it.code] = newList
+		mergedCodeMap = append(mergedCodeMap, it.badCode)
+	}
+	sort.Strings(filteredCodeMap)
+	sort.Strings(mergedCodeMap)
+	os.Stderr.WriteString("filtered " + strings.Join(filteredCodeMap, ",") + "\n")
+	os.Stderr.WriteString("merged " + strings.Join(mergedCodeMap, ",") + "\n")
+}
+
+func mergeTags(data map[string][]geosite.Item) {
+	var codeList []string
+	for code := range data {
+		codeList = append(codeList, code)
+	}
+	var cnCodeList []string
+	for _, code := range codeList {
+		codeParts := strings.Split(code, "@")
+		if len(codeParts) != 2 {
+			continue
+		}
+		if codeParts[1] != "cn" {
+			continue
+		}
+		if !strings.HasPrefix(codeParts[0], "category-") {
+			continue
+		}
+		if strings.HasSuffix(codeParts[0], "-cn") || strings.HasSuffix(codeParts[0], "-!cn") {
+			continue
+		}
+		cnCodeList = append(cnCodeList, code)
+	}
+	newMap := make(map[geosite.Item]bool)
+	for _, item := range data["cn"] {
+		newMap[item] = true
+	}
+	for _, code := range cnCodeList {
+		for _, item := range data[code] {
+			newMap[item] = true
+		}
+	}
+	newList := make([]geosite.Item, 0, len(newMap))
+	for item := range newMap {
+		newList = append(newList, item)
+	}
+	data["cn"] = newList
+	println("merged cn categories: " + strings.Join(cnCodeList, ","))
+}
+
+func generate(release *github.RepositoryRelease, output string, cnOutput string, ruleSetOutput string) error {
 	vData, err := download(release)
 	if err != nil {
 		return err
@@ -183,12 +287,38 @@ func generate(release *github.RepositoryRelease, output string, ruleSetOutput st
 	if err != nil {
 		return err
 	}
+	filterTags(domainMap)
+	mergeTags(domainMap)
 	outputPath, _ := filepath.Abs(output)
 	os.Stderr.WriteString("write " + outputPath + "\n")
+	outputFile, err := os.Create(output)
+	if err != nil {
+		return err
+	}
+	defer outputFile.Close()
 	err = geosite.Write(outputFile, domainMap)
 	if err != nil {
 		return err
 	}
+	cnCodes := []string{
+		"cn",
+		"geolocation-!cn",
+		"category-companies@cn",
+		"apple-update",
+	}
+	cnDomainMap := make(map[string][]geosite.Item)
+	for _, cnCode := range cnCodes {
+		cnDomainMap[cnCode] = domainMap[cnCode]
+	}
+	cnOutputFile, err := os.Create(cnOutput)
+	if err != nil {
+		return err
+	}
+	defer cnOutputFile.Close()
+	err = geosite.Write(cnOutputFile, cnDomainMap)
+	if err != nil {
+		return err
+	}
 	os.RemoveAll(ruleSetOutput)
 	err = os.MkdirAll(ruleSetOutput, 0o755)
 	if err != nil {
@@ -209,7 +339,7 @@ func generate(release *github.RepositoryRelease, output string, ruleSetOutput st
 			},
 		}
 		srsPath, _ := filepath.Abs(filepath.Join(ruleSetOutput, "geosite-"+code+".srs"))
-		os.Stderr.WriteString("write " + srsPath + "\n")
+		//os.Stderr.WriteString("write " + srsPath + "\n")
 		outputRuleSet, err := os.Create(srsPath)
 		if err != nil {
 			return err
@@ -228,7 +358,7 @@ func setActionOutput(name string, content string) {
 	os.Stdout.WriteString("::set-output name=" + name + "::" + content + "\n")
 }
 
-func release(source string, destination string, output string, ruleSetOutput string) error {
+func release(source string, destination string, output string, cnOutput string, ruleSetOutput string) error {
 	sourceRelease, err := fetch(source)
 	if err != nil {
 		return err
@@ -243,7 +373,7 @@ func release(source string, destination string, output string, ruleSetOutput str
 			return nil
 		}
 	}
-	err = generate(sourceRelease, output, ruleSetOutput)
+	err = generate(sourceRelease, output, cnOutput, ruleSetOutput)
 	if err != nil {
 		return err
 	}
@@ -252,7 +382,13 @@ func release(source string, destination string, output string, ruleSetOutput str
 }
 
 func main() {
-	err := release("v2fly/domain-list-community", "sagernet/sing-geosite", "geosite.db", "rule-set")
+	err := release(
+		"v2fly/domain-list-community",
+		"sagernet/sing-geosite",
+		"geosite.db",
+		"geosite-cn.db",
+		"rule-set",
+	)
 	if err != nil {
 		log.Fatal(err)
 	}