wcbing/wcbing-apt-repo
1
0
Fork 0
mirror of https://github.com/wcbing/wcbing-apt-repo.git synced 2025-12-29 02:28:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: github releases, arch name, multithreading

Browse Source 
rewrite github downloader and configuration to store version tag in sqlite.
add 'all' arch, change 'x86_64' to 'amd64'.
add multithreading for downloader.
rename init_deb.py to init-deb.py
This commit is contained in:
wcbing 2025-02-22 21:12:33 +08:00
parent 07d25c28c7
commit 2012302cc4
6 changed files with 178 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,3 +1,4 @@
data/github-local.json data/github-local.json
data/deb.db data/deb.db
deb/ deb/
__pycache__/

97
check_downloader.py
View File

@ -2,9 +2,16 @@
import subprocess import subprocess
import os import os
import sqlite3 import sqlite3
import sys
import logging import logging
from threading import Lock
BASE_DIR = "deb"
DB_DIR = "data"
USER_AGENT = "Debian APT-HTTP/1.3 (2.6.1)" # from Debian 12
version_lock = Lock()
base_dir = "deb"
logging.basicConfig( logging.basicConfig(
format="%(asctime)s %(message)s", format="%(asctime)s %(message)s",
datefmt="%Y/%m/%d %H:%M:%S", datefmt="%Y/%m/%d %H:%M:%S",
@ -13,68 +20,58 @@ logging.basicConfig(
def download(url): def download(url):
file_dir = os.path.join(base_dir, os.path.dirname(url)) """Download file using curl with APT User-Agent."""
if not os.path.exists(file_dir): file_path = os.path.join(BASE_DIR, url.split("?")[0])
os.makedirs(file_dir) os.makedirs(os.path.dirname(file_path), exist_ok=True)
file_path = os.path.join(base_dir, url.split("?")[0]) subprocess.run(["curl", "-H", f"User-Agent: {USER_AGENT}", "-fsLo", file_path, url])
# 用 curl 模拟 apt 下载文件，User-Agent 来自 Debian 12
subprocess.run(
[
"curl",
"-H",
"User-Agent: Debian APT-HTTP/1.3 (2.6.1)",
"-fsLo",
file_path,
url,
]
)
def check_download(name, version, url, arch): def check_download(name, version, url, arch="amd64"):
"""Check and handle package download/update."""
logging.info("%s:%s = %s", name, arch, version) logging.info("%s:%s = %s", name, arch, version)
# connect to db db_path = os.path.join("data", f"{BASE_DIR}.db")
with sqlite3.connect(os.path.join("data", f"{base_dir}.db")) as conn: # get local version
cur = conn.cursor() with version_lock, sqlite3.connect(db_path) as conn:
res = cur.execute( res = conn.execute(
f"SELECT version, url FROM {arch} WHERE name = ?", (name,) f"SELECT version, url FROM '{arch}' WHERE name = ?", (name,)
).fetchall() ).fetchone()
if len(res): if res:
local_version = res[0][0] local_version, local_url = res
local_url = res[0][1] if local_version != version:
if local_version != version: print(f"Update: {name}:{arch} ({local_version} -> {version})")
print(f"Update: {name}:{arch} ({local_version} -> {version})") download(url)
download(url) # update database
# wirte to db with version_lock, sqlite3.connect(db_path) as conn:
cur.execute( conn.execute(
f"UPDATE {arch} SET version = ?, url = ? WHERE name = ?", f"UPDATE '{arch}' SET version = ?, url = ? WHERE name = ?",
(version, url, name), (version, url, name),
) )
# remove old version conn.commit()
if local_url != url: # 针对固定下载链接 # remove old version
old_file_path = os.path.join(base_dir, local_url.split("?")[0]) if local_url != url: # 防止固定下载链接
if os.path.exists(old_file_path): old_file_path = os.path.join(BASE_DIR, local_url.split("?")[0])
os.remove(old_file_path) if os.path.exists(old_file_path):
else: os.remove(old_file_path)
print(f"AddNew: {name}:{arch} ({version})") else:
download(url) print(f"AddNew: {name}:{arch} ({version})")
# wirte to db download(url)
cur.execute( # update database
f"INSERT INTO {arch}(name, version, url) VALUES (?, ?, ?)", with version_lock, sqlite3.connect(db_path) as conn:
conn.execute(
f"INSERT INTO '{arch}'(name, version, url) VALUES (?, ?, ?)",
(name, version, url), (name, version, url),
) )
conn.commit() conn.commit()
if __name__ == "__main__": if __name__ == "__main__":
args = os.sys.argv args = sys.argv
if len(args) == 5: if len(args) in (4, 5):
check_download(args[1], args[2], args[3], args[4]) check_download(*args[1:])
elif len(args) == 4:
check_download(args[1], args[2], args[3], "x86_64")
elif len(args) > 1: elif len(args) > 1:
logging.error(f"Unknown Args: {args[1:]}") logging.error(f"Unknown Args: {args[1:]}")
else: else:
print(f"Usage: {args[0]} <package_name> <version> <url> [arch]") print(f"Usage: {args[0]} <package_name> <version> <url> [arch]")
print("options:") print("options:")
print(" arch: x86_64, arm64. default is x86_64") print(" arch: amd64, arm64, all. default is amd64")

116
data/github.json
View File

@ -1,108 +1,108 @@
{ {
"clash-verge": { "clash-verge": {
"repo": "clash-verge-rev/clash-verge-rev", "repo": "clash-verge-rev/clash-verge-rev",
"file_list": [ "file_list": {
"Clash.Verge_{stripped_version}_amd64.deb", "amd64": "Clash.Verge_{stripped_version}_amd64.deb",
"Clash.Verge_{stripped_version}_arm64.deb" "arm64": "Clash.Verge_{stripped_version}_arm64.deb"
] }
}, },
"mihomo": { "mihomo": {
"repo": "MetaCubeX/mihomo", "repo": "MetaCubeX/mihomo",
"file_list": [ "file_list": {
"mihomo-linux-amd64-compatible-{version_tag}.deb", "amd64": "mihomo-linux-amd64-compatible-{version_tag}.deb",
"mihomo-linux-arm64-{version_tag}.deb" "arm64": "mihomo-linux-arm64-{version_tag}.deb"
] }
}, },
"flclash": { "flclash": {
"repo": "chen08209/FlClash", "repo": "chen08209/FlClash",
"file_list": [ "file_list": {
"FlClash-{stripped_version}-linux-amd64.deb" "amd64": "FlClash-{stripped_version}-linux-amd64.deb"
] }
}, },
"hugo": { "hugo": {
"repo": "gohugoio/hugo", "repo": "gohugoio/hugo",
"file_list": [ "file_list": {
"hugo_extended_{stripped_version}_linux-amd64.deb", "amd64": "hugo_extended_{stripped_version}_linux-amd64.deb",
"hugo_extended_{stripped_version}_linux-arm64.deb" "arm64": "hugo_extended_{stripped_version}_linux-arm64.deb"
] }
}, },
"rustdesk": { "rustdesk": {
"repo": "rustdesk/rustdesk", "repo": "rustdesk/rustdesk",
"file_list": [ "file_list": {
"rustdesk-{version_tag}-x86_64.deb", "amd64": "rustdesk-{version_tag}-x86_64.deb",
"rustdesk-{version_tag}-aarch64.deb" "arm64": "rustdesk-{version_tag}-aarch64.deb"
] }
}, },
"obsidian": { "obsidian": {
"repo": "obsidianmd/obsidian-releases", "repo": "obsidianmd/obsidian-releases",
"file_list": [ "file_list": {
"obsidian_{stripped_version}_amd64.deb" "amd64": "obsidian_{stripped_version}_amd64.deb"
] }
}, },
"tabby": { "tabby": {
"repo": "Eugeny/tabby", "repo": "Eugeny/tabby",
"file_list": [ "file_list": {
"tabby-{stripped_version}-linux-x64.deb", "amd64": "tabby-{stripped_version}-linux-x64.deb",
"tabby-{stripped_version}-linux-arm64.deb" "arm64": "tabby-{stripped_version}-linux-arm64.deb"
] }
}, },
"pandoc": { "pandoc": {
"repo": "jgm/pandoc", "repo": "jgm/pandoc",
"file_list": [ "file_list": {
"pandoc-{version_tag}-1-amd64.deb", "amd64": "pandoc-{version_tag}-1-amd64.deb",
"pandoc-{version_tag}-1-arm64.deb" "arm64": "pandoc-{version_tag}-1-arm64.deb"
] }
}, },
"localsend": { "localsend": {
"repo": "localsend/localsend", "repo": "localsend/localsend",
"file_list": [ "file_list": {
"LocalSend-{stripped_version}-linux-x86-64.deb", "amd64": "LocalSend-{stripped_version}-linux-x86-64.deb",
"LocalSend-{stripped_version}-linux-arm-64.deb" "arm64": "LocalSend-{stripped_version}-linux-arm-64.deb"
] }
}, },
"motrix": { "motrix": {
"repo": "agalwood/Motrix", "repo": "agalwood/Motrix",
"file_list": [ "file_list": {
"Motrix_{stripped_version}_amd64.deb", "amd64": "Motrix_{stripped_version}_amd64.deb",
"Motrix_{stripped_version}_arm64.deb" "arm64": "Motrix_{stripped_version}_arm64.deb"
] }
}, },
"peazip": { "peazip": {
"repo": "peazip/PeaZip", "repo": "peazip/PeaZip",
"file_list": [ "file_list": {
"peazip_{version_tag}.LINUX.GTK2-1_amd64.deb" "amd64": "peazip_{version_tag}.LINUX.GTK2-1_amd64.deb"
] }
}, },
"neovim": { "neovim": {
"repo": "neovim/neovim-releases", "repo": "neovim/neovim-releases",
"file_list": [ "file_list": {
"nvim-linux64.deb" "amd64": "nvim-linux64.deb"
] }
}, },
"hiddify": { "hiddify": {
"repo": "hiddify/hiddify-app", "repo": "hiddify/hiddify-app",
"file_list": [ "file_list": {
"Hiddify-Debian-x64.deb" "amd64": "Hiddify-Debian-x64.deb"
] }
}, },
"cloudflared": { "cloudflared": {
"repo": "cloudflare/cloudflared", "repo": "cloudflare/cloudflared",
"file_list": [ "file_list": {
"cloudflared-linux-amd64.deb", "amd64": "cloudflared-linux-amd64.deb",
"cloudflared-linux-arm64.deb" "arm64": "cloudflared-linux-arm64.deb"
] }
}, },
"caddy": { "caddy": {
"repo": "caddyserver/caddy", "repo": "caddyserver/caddy",
"file_list": [ "file_list": {
"caddy_{stripped_version}_linux_amd64.deb", "amd64": "caddy_{stripped_version}_linux_amd64.deb",
"caddy_{stripped_version}_linux_arm64.deb" "arm64": "caddy_{stripped_version}_linux_arm64.deb"
] }
}, },
"foliate": { "foliate": {
"repo": "johnfactotum/foliate", "repo": "johnfactotum/foliate",
"file_list": [ "file_list": {
"foliate_{version_tag}_all.deb" "all": "foliate_{version_tag}_all.deb"
] }
} }
} }

60
get-github-releases.py Executable file
View File

@ -0,0 +1,60 @@
#!/usr/bin/env python3
import argparse
import requests
import json
import os
import re
from concurrent.futures import ThreadPoolExecutor, wait
from check_downloader import check_download
github_info_list = {}
CONFIG = {"data_dir": "data", "proxy": "", "thread": 5}
# 读取命令行参数
def read_args():
parser = argparse.ArgumentParser()
parser.add_argument("-d", "--data", default="data", help="从 <DATA> 读取仓库配置")
parser.add_argument(
"-p", "--proxy", default="", help="Github 代理，<PROXY> 必须以 / 结尾"
)
parser.add_argument(
"-t", "--thread", type=int, default=5, help="并发下载线程数量，默认为 5"
)
args = parser.parse_args()
CONFIG.update({"data_dir": args.data, "proxy": args.proxy, "thread": args.thread})
if __name__ == "__main__":
read_args()
# read all repo info 读取所有仓库配置
with open(os.path.join(CONFIG["data_dir"], "github.json"), "r") as f:
github_info_list = json.load(f)
tasks = []
with ThreadPoolExecutor(max_workers=CONFIG["thread"]) as executor:
for name, repo in github_info_list.items():
release_url = (
f'{CONFIG["proxy"]}https://github.com/{repo["repo"]}/releases'
)
# get latest version tag 获取最新版本标签
location = requests.head(release_url + "/latest").headers.get("Location", "")
match = re.search(r".*releases/tag/([^/]+)", location)
if not match:
continue
version_tag = match.group(1)
version = version_tag[1:] if version_tag[0].lower() == "v" else version_tag
for arch, file_name in repo["file_list"].items():
release_file = file_name.format(
version_tag=version_tag, stripped_version=version
)
file_url = f"{release_url}/download/{version_tag}/{release_file}"
# 提交任务到线程池
tasks.append(
executor.submit(check_download, name, version, file_url, arch)
)
# 等待所有任务完成
wait(tasks)

11
init_deb.py → init-deb.py
View File

@ -6,7 +6,7 @@ import sqlite3
conn = sqlite3.connect("data/deb.db") conn = sqlite3.connect("data/deb.db")
conn.execute( conn.execute(
""" """
CREATE TABLE IF NOT EXISTS x86_64 ( CREATE TABLE IF NOT EXISTS amd64 (
name TEXT UNIQUE, name TEXT UNIQUE,
version TEXT, version TEXT,
url TEXT url TEXT
@ -22,6 +22,15 @@ conn.execute(
); );
""" """
) )
conn.execute(
"""
CREATE TABLE IF NOT EXISTS 'all' (
name TEXT UNIQUE,
version TEXT,
url TEXT
);
"""
)
conn.commit() conn.commit()
conn.close() conn.close()

2
run.sh
View File

@ -7,7 +7,7 @@ gen_release() {
} }
# check for updates # check for updates
$HOME/go/bin/github-downloader -r -o deb ./get-github-releases.py
find get -type f -name "*.sh" -exec sh {} \; find get -type f -name "*.sh" -exec sh {} \;
cd deb cd deb