From 2012302cc4f4e1a6c6410e7f47964732b5af7553 Mon Sep 17 00:00:00 2001 From: wcbing Date: Sat, 22 Feb 2025 21:12:33 +0800 Subject: [PATCH] refactor: github releases, arch name, multithreading rewrite github downloader and configuration to store version tag in sqlite. add 'all' arch, change 'x86_64' to 'amd64'. add multithreading for downloader. rename init_deb.py to init-deb.py --- .gitignore | 3 +- check_downloader.py | 97 +++++++++++++++---------------- data/github.json | 116 ++++++++++++++++++------------------- get-github-releases.py | 60 +++++++++++++++++++ init_deb.py => init-deb.py | 11 +++- run.sh | 2 +- 6 files changed, 178 insertions(+), 111 deletions(-) create mode 100755 get-github-releases.py rename init_deb.py => init-deb.py (66%) diff --git a/.gitignore b/.gitignore index 9b2f439..cb91354 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ data/github-local.json data/deb.db -deb/ \ No newline at end of file +deb/ +__pycache__/ \ No newline at end of file diff --git a/check_downloader.py b/check_downloader.py index 531d48f..eb40abb 100755 --- a/check_downloader.py +++ b/check_downloader.py @@ -2,9 +2,16 @@ import subprocess import os import sqlite3 +import sys import logging +from threading import Lock + +BASE_DIR = "deb" +DB_DIR = "data" +USER_AGENT = "Debian APT-HTTP/1.3 (2.6.1)" # from Debian 12 + +version_lock = Lock() -base_dir = "deb" logging.basicConfig( format="%(asctime)s %(message)s", datefmt="%Y/%m/%d %H:%M:%S", @@ -13,68 +20,58 @@ logging.basicConfig( def download(url): - file_dir = os.path.join(base_dir, os.path.dirname(url)) - if not os.path.exists(file_dir): - os.makedirs(file_dir) - file_path = os.path.join(base_dir, url.split("?")[0]) - # 用 curl 模拟 apt 下载文件，User-Agent 来自 Debian 12 - subprocess.run( - [ - "curl", - "-H", - "User-Agent: Debian APT-HTTP/1.3 (2.6.1)", - "-fsLo", - file_path, - url, - ] - ) + """Download file using curl with APT User-Agent.""" + file_path = os.path.join(BASE_DIR, url.split("?")[0]) + os.makedirs(os.path.dirname(file_path), exist_ok=True) + subprocess.run(["curl", "-H", f"User-Agent: {USER_AGENT}", "-fsLo", file_path, url]) -def check_download(name, version, url, arch): +def check_download(name, version, url, arch="amd64"): + """Check and handle package download/update.""" logging.info("%s:%s = %s", name, arch, version) - # connect to db - with sqlite3.connect(os.path.join("data", f"{base_dir}.db")) as conn: - cur = conn.cursor() - res = cur.execute( - f"SELECT version, url FROM {arch} WHERE name = ?", (name,) - ).fetchall() - if len(res): - local_version = res[0][0] - local_url = res[0][1] - if local_version != version: - print(f"Update: {name}:{arch} ({local_version} -> {version})") - download(url) - # wirte to db - cur.execute( - f"UPDATE {arch} SET version = ?, url = ? WHERE name = ?", + db_path = os.path.join("data", f"{BASE_DIR}.db") + # get local version + with version_lock, sqlite3.connect(db_path) as conn: + res = conn.execute( + f"SELECT version, url FROM '{arch}' WHERE name = ?", (name,) + ).fetchone() + if res: + local_version, local_url = res + if local_version != version: + print(f"Update: {name}:{arch} ({local_version} -> {version})") + download(url) + # update database + with version_lock, sqlite3.connect(db_path) as conn: + conn.execute( + f"UPDATE '{arch}' SET version = ?, url = ? WHERE name = ?", (version, url, name), ) - # remove old version - if local_url != url: # 针对固定下载链接 - old_file_path = os.path.join(base_dir, local_url.split("?")[0]) - if os.path.exists(old_file_path): - os.remove(old_file_path) - else: - print(f"AddNew: {name}:{arch} ({version})") - download(url) - # wirte to db - cur.execute( - f"INSERT INTO {arch}(name, version, url) VALUES (?, ?, ?)", + conn.commit() + # remove old version + if local_url != url: # 防止固定下载链接 + old_file_path = os.path.join(BASE_DIR, local_url.split("?")[0]) + if os.path.exists(old_file_path): + os.remove(old_file_path) + else: + print(f"AddNew: {name}:{arch} ({version})") + download(url) + # update database + with version_lock, sqlite3.connect(db_path) as conn: + conn.execute( + f"INSERT INTO '{arch}'(name, version, url) VALUES (?, ?, ?)", (name, version, url), ) - conn.commit() + conn.commit() if __name__ == "__main__": - args = os.sys.argv - if len(args) == 5: - check_download(args[1], args[2], args[3], args[4]) - elif len(args) == 4: - check_download(args[1], args[2], args[3], "x86_64") + args = sys.argv + if len(args) in (4, 5): + check_download(*args[1:]) elif len(args) > 1: logging.error(f"Unknown Args: {args[1:]}") else: print(f"Usage: {args[0]} [arch]") print("options:") - print(" arch: x86_64, arm64. default is x86_64") + print(" arch: amd64, arm64, all. default is amd64") diff --git a/data/github.json b/data/github.json index 631b59c..8275b6f 100644 --- a/data/github.json +++ b/data/github.json @@ -1,108 +1,108 @@ { "clash-verge": { "repo": "clash-verge-rev/clash-verge-rev", - "file_list": [ - "Clash.Verge_{stripped_version}_amd64.deb", - "Clash.Verge_{stripped_version}_arm64.deb" - ] + "file_list": { + "amd64": "Clash.Verge_{stripped_version}_amd64.deb", + "arm64": "Clash.Verge_{stripped_version}_arm64.deb" + } }, "mihomo": { "repo": "MetaCubeX/mihomo", - "file_list": [ - "mihomo-linux-amd64-compatible-{version_tag}.deb", - "mihomo-linux-arm64-{version_tag}.deb" - ] + "file_list": { + "amd64": "mihomo-linux-amd64-compatible-{version_tag}.deb", + "arm64": "mihomo-linux-arm64-{version_tag}.deb" + } }, "flclash": { "repo": "chen08209/FlClash", - "file_list": [ - "FlClash-{stripped_version}-linux-amd64.deb" - ] + "file_list": { + "amd64": "FlClash-{stripped_version}-linux-amd64.deb" + } }, "hugo": { "repo": "gohugoio/hugo", - "file_list": [ - "hugo_extended_{stripped_version}_linux-amd64.deb", - "hugo_extended_{stripped_version}_linux-arm64.deb" - ] + "file_list": { + "amd64": "hugo_extended_{stripped_version}_linux-amd64.deb", + "arm64": "hugo_extended_{stripped_version}_linux-arm64.deb" + } }, "rustdesk": { "repo": "rustdesk/rustdesk", - "file_list": [ - "rustdesk-{version_tag}-x86_64.deb", - "rustdesk-{version_tag}-aarch64.deb" - ] + "file_list": { + "amd64": "rustdesk-{version_tag}-x86_64.deb", + "arm64": "rustdesk-{version_tag}-aarch64.deb" + } }, "obsidian": { "repo": "obsidianmd/obsidian-releases", - "file_list": [ - "obsidian_{stripped_version}_amd64.deb" - ] + "file_list": { + "amd64": "obsidian_{stripped_version}_amd64.deb" + } }, "tabby": { "repo": "Eugeny/tabby", - "file_list": [ - "tabby-{stripped_version}-linux-x64.deb", - "tabby-{stripped_version}-linux-arm64.deb" - ] + "file_list": { + "amd64": "tabby-{stripped_version}-linux-x64.deb", + "arm64": "tabby-{stripped_version}-linux-arm64.deb" + } }, "pandoc": { "repo": "jgm/pandoc", - "file_list": [ - "pandoc-{version_tag}-1-amd64.deb", - "pandoc-{version_tag}-1-arm64.deb" - ] + "file_list": { + "amd64": "pandoc-{version_tag}-1-amd64.deb", + "arm64": "pandoc-{version_tag}-1-arm64.deb" + } }, "localsend": { "repo": "localsend/localsend", - "file_list": [ - "LocalSend-{stripped_version}-linux-x86-64.deb", - "LocalSend-{stripped_version}-linux-arm-64.deb" - ] + "file_list": { + "amd64": "LocalSend-{stripped_version}-linux-x86-64.deb", + "arm64": "LocalSend-{stripped_version}-linux-arm-64.deb" + } }, "motrix": { "repo": "agalwood/Motrix", - "file_list": [ - "Motrix_{stripped_version}_amd64.deb", - "Motrix_{stripped_version}_arm64.deb" - ] + "file_list": { + "amd64": "Motrix_{stripped_version}_amd64.deb", + "arm64": "Motrix_{stripped_version}_arm64.deb" + } }, "peazip": { "repo": "peazip/PeaZip", - "file_list": [ - "peazip_{version_tag}.LINUX.GTK2-1_amd64.deb" - ] + "file_list": { + "amd64": "peazip_{version_tag}.LINUX.GTK2-1_amd64.deb" + } }, "neovim": { "repo": "neovim/neovim-releases", - "file_list": [ - "nvim-linux64.deb" - ] + "file_list": { + "amd64": "nvim-linux64.deb" + } }, "hiddify": { "repo": "hiddify/hiddify-app", - "file_list": [ - "Hiddify-Debian-x64.deb" - ] + "file_list": { + "amd64": "Hiddify-Debian-x64.deb" + } }, "cloudflared": { "repo": "cloudflare/cloudflared", - "file_list": [ - "cloudflared-linux-amd64.deb", - "cloudflared-linux-arm64.deb" - ] + "file_list": { + "amd64": "cloudflared-linux-amd64.deb", + "arm64": "cloudflared-linux-arm64.deb" + } }, "caddy": { "repo": "caddyserver/caddy", - "file_list": [ - "caddy_{stripped_version}_linux_amd64.deb", - "caddy_{stripped_version}_linux_arm64.deb" - ] + "file_list": { + "amd64": "caddy_{stripped_version}_linux_amd64.deb", + "arm64": "caddy_{stripped_version}_linux_arm64.deb" + } }, "foliate": { "repo": "johnfactotum/foliate", - "file_list": [ - "foliate_{version_tag}_all.deb" - ] + "file_list": { + "all": "foliate_{version_tag}_all.deb" + } } } \ No newline at end of file diff --git a/get-github-releases.py b/get-github-releases.py new file mode 100755 index 0000000..19ffd21 --- /dev/null +++ b/get-github-releases.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +import argparse +import requests +import json +import os +import re +from concurrent.futures import ThreadPoolExecutor, wait +from check_downloader import check_download + +github_info_list = {} + +CONFIG = {"data_dir": "data", "proxy": "", "thread": 5} + + +# 读取命令行参数 +def read_args(): + parser = argparse.ArgumentParser() + parser.add_argument("-d", "--data", default="data", help="从 读取仓库配置") + parser.add_argument( + "-p", "--proxy", default="", help="Github 代理， 必须以 / 结尾" + ) + parser.add_argument( + "-t", "--thread", type=int, default=5, help="并发下载线程数量，默认为 5" + ) + args = parser.parse_args() + CONFIG.update({"data_dir": args.data, "proxy": args.proxy, "thread": args.thread}) + + +if __name__ == "__main__": + read_args() + + # read all repo info 读取所有仓库配置 + with open(os.path.join(CONFIG["data_dir"], "github.json"), "r") as f: + github_info_list = json.load(f) + + tasks = [] + with ThreadPoolExecutor(max_workers=CONFIG["thread"]) as executor: + for name, repo in github_info_list.items(): + release_url = ( + f'{CONFIG["proxy"]}https://github.com/{repo["repo"]}/releases' + ) + # get latest version tag 获取最新版本标签 + location = requests.head(release_url + "/latest").headers.get("Location", "") + match = re.search(r".*releases/tag/([^/]+)", location) + if not match: + continue + version_tag = match.group(1) + version = version_tag[1:] if version_tag[0].lower() == "v" else version_tag + + for arch, file_name in repo["file_list"].items(): + release_file = file_name.format( + version_tag=version_tag, stripped_version=version + ) + file_url = f"{release_url}/download/{version_tag}/{release_file}" + # 提交任务到线程池 + tasks.append( + executor.submit(check_download, name, version, file_url, arch) + ) + # 等待所有任务完成 + wait(tasks) diff --git a/init_deb.py b/init-deb.py similarity index 66% rename from init_deb.py rename to init-deb.py index 1ac0353..4e6ae83 100755 --- a/init_deb.py +++ b/init-deb.py @@ -6,7 +6,7 @@ import sqlite3 conn = sqlite3.connect("data/deb.db") conn.execute( """ - CREATE TABLE IF NOT EXISTS x86_64 ( + CREATE TABLE IF NOT EXISTS amd64 ( name TEXT UNIQUE, version TEXT, url TEXT @@ -22,6 +22,15 @@ conn.execute( ); """ ) +conn.execute( + """ + CREATE TABLE IF NOT EXISTS 'all' ( + name TEXT UNIQUE, + version TEXT, + url TEXT + ); + """ +) conn.commit() conn.close() diff --git a/run.sh b/run.sh index cf09187..19a9a36 100755 --- a/run.sh +++ b/run.sh @@ -7,7 +7,7 @@ gen_release() { } # check for updates -$HOME/go/bin/github-downloader -r -o deb +./get-github-releases.py find get -type f -name "*.sh" -exec sh {} \; cd deb