diff --git a/.gitignore b/.gitignore index 9b2f439..cb91354 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ data/github-local.json data/deb.db -deb/ \ No newline at end of file +deb/ +__pycache__/ \ No newline at end of file diff --git a/check_downloader.py b/check_downloader.py index 531d48f..a1d71d4 100755 --- a/check_downloader.py +++ b/check_downloader.py @@ -2,9 +2,16 @@ import subprocess import os import sqlite3 +import sys import logging +from threading import Lock + +BASE_DIR = "deb" +DB_DIR = "data" +USER_AGENT = "Debian APT-HTTP/1.3 (2.6.1)" # from Debian 12 + +version_lock = Lock() -base_dir = "deb" logging.basicConfig( format="%(asctime)s %(message)s", datefmt="%Y/%m/%d %H:%M:%S", @@ -12,69 +19,59 @@ logging.basicConfig( ) -def download(url): - file_dir = os.path.join(base_dir, os.path.dirname(url)) - if not os.path.exists(file_dir): - os.makedirs(file_dir) - file_path = os.path.join(base_dir, url.split("?")[0]) - # 用 curl 模拟 apt 下载文件，User-Agent 来自 Debian 12 - subprocess.run( - [ - "curl", - "-H", - "User-Agent: Debian APT-HTTP/1.3 (2.6.1)", - "-fsLo", - file_path, - url, - ] - ) +def download(url: str) -> None: + """Download file using curl with APT User-Agent.""" + file_path = os.path.join(BASE_DIR, url.split("?")[0]) + os.makedirs(os.path.dirname(file_path), exist_ok=True) + subprocess.run(["curl", "-H", f"User-Agent: {USER_AGENT}", "-fsLo", file_path, url]) -def check_download(name, version, url, arch): +def check_download(name: str, version: str, url: str, arch: str="amd64") -> None: + """Check and handle package download/update.""" logging.info("%s:%s = %s", name, arch, version) - # connect to db - with sqlite3.connect(os.path.join("data", f"{base_dir}.db")) as conn: - cur = conn.cursor() - res = cur.execute( - f"SELECT version, url FROM {arch} WHERE name = ?", (name,) - ).fetchall() - if len(res): - local_version = res[0][0] - local_url = res[0][1] - if local_version != version: - print(f"Update: {name}:{arch} ({local_version} -> {version})") - download(url) - # wirte to db - cur.execute( - f"UPDATE {arch} SET version = ?, url = ? WHERE name = ?", + db_path = os.path.join("data", f"{BASE_DIR}.db") + # get local version + with version_lock, sqlite3.connect(db_path) as conn: + res = conn.execute( + f"SELECT version, url FROM '{arch}' WHERE name = ?", (name,) + ).fetchone() + if res: + local_version, local_url = res + if local_version != version: + print(f"Update: {name}:{arch} ({local_version} -> {version})") + download(url) + # update database + with version_lock, sqlite3.connect(db_path) as conn: + conn.execute( + f"UPDATE '{arch}' SET version = ?, url = ? WHERE name = ?", (version, url, name), ) - # remove old version - if local_url != url: # 针对固定下载链接 - old_file_path = os.path.join(base_dir, local_url.split("?")[0]) - if os.path.exists(old_file_path): - os.remove(old_file_path) - else: - print(f"AddNew: {name}:{arch} ({version})") - download(url) - # wirte to db - cur.execute( - f"INSERT INTO {arch}(name, version, url) VALUES (?, ?, ?)", + conn.commit() + # remove old version + if local_url != url: # 防止固定下载链接 + old_file_path = os.path.join(BASE_DIR, local_url.split("?")[0]) + if os.path.exists(old_file_path): + os.remove(old_file_path) + else: + print(f"AddNew: {name}:{arch} ({version})") + download(url) + # update database + with version_lock, sqlite3.connect(db_path) as conn: + conn.execute( + f"INSERT INTO '{arch}'(name, version, url) VALUES (?, ?, ?)", (name, version, url), ) - conn.commit() + conn.commit() if __name__ == "__main__": - args = os.sys.argv - if len(args) == 5: - check_download(args[1], args[2], args[3], args[4]) - elif len(args) == 4: - check_download(args[1], args[2], args[3], "x86_64") + args = sys.argv + if len(args) in (4, 5): + check_download(*args[1:]) elif len(args) > 1: logging.error(f"Unknown Args: {args[1:]}") else: print(f"Usage: {args[0]} [arch]") print("options:") - print(" arch: x86_64, arm64. default is x86_64") + print(" arch: amd64, arm64, all. default is amd64") diff --git a/data/github.json b/data/github.json index 631b59c..adadc13 100644 --- a/data/github.json +++ b/data/github.json @@ -1,108 +1,108 @@ { "clash-verge": { "repo": "clash-verge-rev/clash-verge-rev", - "file_list": [ - "Clash.Verge_{stripped_version}_amd64.deb", - "Clash.Verge_{stripped_version}_arm64.deb" - ] + "file_list": { + "amd64": "Clash.Verge_{version}_amd64.deb", + "arm64": "Clash.Verge_{version}_arm64.deb" + } }, "mihomo": { "repo": "MetaCubeX/mihomo", - "file_list": [ - "mihomo-linux-amd64-compatible-{version_tag}.deb", - "mihomo-linux-arm64-{version_tag}.deb" - ] + "file_list": { + "amd64": "mihomo-linux-amd64-compatible-{releases_tag}.deb", + "arm64": "mihomo-linux-arm64-{releases_tag}.deb" + } }, "flclash": { "repo": "chen08209/FlClash", - "file_list": [ - "FlClash-{stripped_version}-linux-amd64.deb" - ] + "file_list": { + "amd64": "FlClash-{version}-linux-amd64.deb" + } }, "hugo": { "repo": "gohugoio/hugo", - "file_list": [ - "hugo_extended_{stripped_version}_linux-amd64.deb", - "hugo_extended_{stripped_version}_linux-arm64.deb" - ] + "file_list": { + "amd64": "hugo_extended_{version}_linux-amd64.deb", + "arm64": "hugo_extended_{version}_linux-arm64.deb" + } }, "rustdesk": { "repo": "rustdesk/rustdesk", - "file_list": [ - "rustdesk-{version_tag}-x86_64.deb", - "rustdesk-{version_tag}-aarch64.deb" - ] + "file_list": { + "amd64": "rustdesk-{releases_tag}-x86_64.deb", + "arm64": "rustdesk-{releases_tag}-aarch64.deb" + } }, "obsidian": { "repo": "obsidianmd/obsidian-releases", - "file_list": [ - "obsidian_{stripped_version}_amd64.deb" - ] + "file_list": { + "amd64": "obsidian_{version}_amd64.deb" + } }, "tabby": { "repo": "Eugeny/tabby", - "file_list": [ - "tabby-{stripped_version}-linux-x64.deb", - "tabby-{stripped_version}-linux-arm64.deb" - ] + "file_list": { + "amd64": "tabby-{version}-linux-x64.deb", + "arm64": "tabby-{version}-linux-arm64.deb" + } }, "pandoc": { "repo": "jgm/pandoc", - "file_list": [ - "pandoc-{version_tag}-1-amd64.deb", - "pandoc-{version_tag}-1-arm64.deb" - ] + "file_list": { + "amd64": "pandoc-{releases_tag}-1-amd64.deb", + "arm64": "pandoc-{releases_tag}-1-arm64.deb" + } }, "localsend": { "repo": "localsend/localsend", - "file_list": [ - "LocalSend-{stripped_version}-linux-x86-64.deb", - "LocalSend-{stripped_version}-linux-arm-64.deb" - ] + "file_list": { + "amd64": "LocalSend-{version}-linux-x86-64.deb", + "arm64": "LocalSend-{version}-linux-arm-64.deb" + } }, "motrix": { "repo": "agalwood/Motrix", - "file_list": [ - "Motrix_{stripped_version}_amd64.deb", - "Motrix_{stripped_version}_arm64.deb" - ] + "file_list": { + "amd64": "Motrix_{version}_amd64.deb", + "arm64": "Motrix_{version}_arm64.deb" + } }, "peazip": { "repo": "peazip/PeaZip", - "file_list": [ - "peazip_{version_tag}.LINUX.GTK2-1_amd64.deb" - ] + "file_list": { + "amd64": "peazip_{releases_tag}.LINUX.GTK2-1_amd64.deb" + } }, "neovim": { "repo": "neovim/neovim-releases", - "file_list": [ - "nvim-linux64.deb" - ] + "file_list": { + "amd64": "nvim-linux64.deb" + } }, "hiddify": { "repo": "hiddify/hiddify-app", - "file_list": [ - "Hiddify-Debian-x64.deb" - ] + "file_list": { + "amd64": "Hiddify-Debian-x64.deb" + } }, "cloudflared": { "repo": "cloudflare/cloudflared", - "file_list": [ - "cloudflared-linux-amd64.deb", - "cloudflared-linux-arm64.deb" - ] + "file_list": { + "amd64": "cloudflared-linux-amd64.deb", + "arm64": "cloudflared-linux-arm64.deb" + } }, "caddy": { "repo": "caddyserver/caddy", - "file_list": [ - "caddy_{stripped_version}_linux_amd64.deb", - "caddy_{stripped_version}_linux_arm64.deb" - ] + "file_list": { + "amd64": "caddy_{version}_linux_amd64.deb", + "arm64": "caddy_{version}_linux_arm64.deb" + } }, "foliate": { "repo": "johnfactotum/foliate", - "file_list": [ - "foliate_{version_tag}_all.deb" - ] + "file_list": { + "all": "foliate_{releases_tag}_all.deb" + } } } \ No newline at end of file diff --git a/data/repo_list.json b/data/repo_list.json index 8316503..911fa24 100644 --- a/data/repo_list.json +++ b/data/repo_list.json @@ -1,106 +1,126 @@ -[ - { - "name": "google-chrome", +{ + "google-chrome": { "repo": "https://dl.google.com/linux/chrome/deb/", - "amd64_path": "dists/stable/main/binary-amd64/Packages.gz" + "path": { + "amd64": "dists/stable/main/binary-amd64/Packages.gz" + } }, - { - "name": "google-earth", + "google-earth": { "repo": "https://dl.google.com/linux/earth/deb/", - "amd64_path": "dists/stable/main/binary-amd64/Packages.gz" + "path": { + "amd64": "dists/stable/main/binary-amd64/Packages.gz" + } }, - { - "name": "termius", + "termius": { "repo": "https://deb.termius.com/", - "amd64_path": "dists/squeeze/main/binary-amd64/Packages.gz" + "path": { + "amd64": "dists/squeeze/main/binary-amd64/Packages.gz" + } }, - { - "name": "steam", + "steam": { "repo": "https://repo.steampowered.com/steam/", - "amd64_path": "dists/stable/steam/binary-amd64/Packages.gz" + "path": { + "amd64": "dists/stable/steam/binary-amd64/Packages.gz" + } }, - { - "name": "firefox", + "firefox": { "repo": "https://packages.mozilla.org/apt/", - "amd64_path": "dists/mozilla/main/binary-amd64/Packages", - "arm64_path": "dists/mozilla/main/binary-arm64/Packages" + "path": { + "amd64": "dists/mozilla/main/binary-amd64/Packages", + "arm64": "dists/mozilla/main/binary-arm64/Packages" + } }, - { - "name": "microsoft-edge", + "microsoft-edge": { "repo": "https://packages.microsoft.com/repos/edge/", - "amd64_path": "dists/stable/main/binary-amd64/Packages.gz" + "path": { + "amd64": "dists/stable/main/binary-amd64/Packages.gz" + } }, - { - "name": "opera-stable", + "opera-stable": { "repo": "https://deb.opera.com/opera-stable/", - "amd64_path": "dists/stable/non-free/binary-amd64/Packages.gz" + "path": { + "amd64": "dists/stable/non-free/binary-amd64/Packages.gz" + } }, - { - "name": "code", + "code": { "repo": "https://packages.microsoft.com/repos/code/", - "amd64_path": "dists/stable/main/binary-amd64/Packages.gz", - "arm64_path": "dists/stable/main/binary-arm64/Packages.gz" + "path": { + "amd64": "dists/stable/main/binary-amd64/Packages.gz", + "arm64": "dists/stable/main/binary-arm64/Packages.gz" + } }, - { - "name": "tailscale", + "tailscale": { "repo": "https://pkgs.tailscale.com/stable/debian/", - "amd64_path": "dists/sid/main/binary-amd64/Packages.gz", - "arm64_path": "dists/sid/main/binary-arm64/Packages.gz" + "path": { + "amd64": "dists/sid/main/binary-amd64/Packages.gz", + "arm64": "dists/sid/main/binary-arm64/Packages.gz" + } }, - { - "name": "sublime", + "sublime": { "repo": "https://download.sublimetext.com/", - "mix_path": "apt/stable/Packages.gz" + "path": { + "mix": "apt/stable/Packages.gz" + } }, - { - "name": "black-desk", + "black-desk": { "repo": "https://github.com/black-desk/debs/releases/latest/download/", - "amd64_path": "Packages" + "path": { + "amd64": "Packages" + } }, - { - "name": "typora", + "typora": { "repo": "https://typoraio.cn/linux/", - "mix_path": "Packages.gz" + "path": { + "mix": "Packages.gz" + } }, - { - "name": "zotero", + "zotero": { "repo": "https://zotero.retorque.re/file/apt-package-archive/", - "amd64_path": "Packages" + "path": { + "amd64": "Packages" + } }, - { - "name": "gh", + "gh": { "repo": "https://cli.github.com/packages/", - "amd64_path": "dists/stable/main/binary-amd64/Packages.gz", - "arm64_path": "dists/stable/main/binary-arm64/Packages.gz" + "path": { + "amd64": "dists/stable/main/binary-amd64/Packages.gz", + "arm64": "dists/stable/main/binary-arm64/Packages.gz" + } }, - { - "name": "dufs", + "dufs": { "repo": "https://github.com/wcbing-build/dufs-debs/releases/latest/download/", - "mix_path": "Packages" + "path": { + "mix": "Packages" + } }, - { - "name": "frp", + "frp": { "repo": "https://github.com/wcbing-build/frp-debs/releases/latest/download/", - "mix_path": "Packages" + "path": { + "mix": "Packages" + } }, - { - "name": "lazydocker", + "lazydocker": { "repo": "https://github.com/wcbing-build/lazydocker-debs/releases/latest/download/", - "mix_path": "Packages" + "path": { + "mix": "Packages" + } }, - { - "name": "lazygit", + "lazygit": { "repo": "https://github.com/wcbing-build/lazygit-debs/releases/latest/download/", - "mix_path": "Packages" + "path": { + "mix": "Packages" + } }, - { - "name": "nexttrace", + "nexttrace": { "repo": "https://github.com/nxtrace/nexttrace-debs/releases/latest/download/", - "mix_path": "Packages" + "path": { + "mix": "Packages" + } }, - { - "name": "debiancn", + "debiancn": { "repo": "https://mirrors.cernet.edu.cn/debiancn/", - "amd64_path": "dists/bookworm/main/binary-amd64/Packages.gz" + "path": { + "amd64": "dists/bookworm/main/binary-amd64/Packages.gz" + } } -] +} \ No newline at end of file diff --git a/get-github-releases.py b/get-github-releases.py new file mode 100755 index 0000000..4962126 --- /dev/null +++ b/get-github-releases.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +import argparse +import requests +import json +import os +import re +from concurrent.futures import ThreadPoolExecutor, wait +from check_downloader import check_download + +github_info_list = {} + +CONFIG = {"data_dir": "data", "proxy": "", "thread": 5} + + +# 读取命令行参数 +def read_args(): + parser = argparse.ArgumentParser() + parser.add_argument("-d", "--data", default="data", help="从 读取仓库配置") + parser.add_argument( + "-p", "--proxy", default="", help="Github 代理， 必须以 / 结尾" + ) + parser.add_argument( + "-t", "--thread", type=int, default=5, help="并发下载线程数量，默认为 5" + ) + args = parser.parse_args() + CONFIG.update({"data_dir": args.data, "proxy": args.proxy, "thread": args.thread}) + + +if __name__ == "__main__": + read_args() + + # read all repo info 读取所有仓库配置 + with open(os.path.join(CONFIG["data_dir"], "github.json"), "r") as f: + github_info_list = json.load(f) + + tasks = [] + with ThreadPoolExecutor(max_workers=CONFIG["thread"]) as executor: + for name, repo in github_info_list.items(): + release_url = ( + f'{CONFIG["proxy"]}https://github.com/{repo["repo"]}/releases' + ) + # get latest releases tag 获取最新版本标签 + location = requests.head(release_url + "/latest").headers.get("Location", "") + match = re.search(r".*releases/tag/([^/]+)", location) + if not match: + continue + releases_tag = match.group(1) + version = match.group() if (match := re.search("[0-9].*", releases_tag)) else "" + + + for arch, file_name in repo["file_list"].items(): + release_file = file_name.format( + releases_tag=releases_tag, version=version + ) + file_url = f"{release_url}/download/{releases_tag}/{release_file}" + # 提交任务到线程池 + tasks.append( + executor.submit(check_download, name, version, file_url, arch) + ) + # 等待所有任务完成 + wait(tasks) diff --git a/init_deb.py b/init_deb.py index 1ac0353..b8d2d57 100755 --- a/init_deb.py +++ b/init_deb.py @@ -4,24 +4,16 @@ import sqlite3 # create table conn = sqlite3.connect("data/deb.db") -conn.execute( - """ - CREATE TABLE IF NOT EXISTS x86_64 ( - name TEXT UNIQUE, - version TEXT, - url TEXT - ); - """ -) -conn.execute( - """ - CREATE TABLE IF NOT EXISTS arm64 ( - name TEXT UNIQUE, - version TEXT, - url TEXT - ); - """ -) +for arch in ["amd64", "arm64", "all"]: + conn.execute( + f""" + CREATE TABLE IF NOT EXISTS '{arch}' ( + name TEXT UNIQUE, + version TEXT, + url TEXT + ); + """ + ) conn.commit() conn.close() diff --git a/merge-apt-repo.py b/merge-apt-repo.py index a001f7c..310d8b8 100755 --- a/merge-apt-repo.py +++ b/merge-apt-repo.py @@ -17,30 +17,37 @@ package_version = {arch: {} for arch in ["all", "amd64", "i386", "arm64"]} package_info = {arch: {} for arch in ["all", "amd64", "i386", "arm64"]} lock = {arch: Lock() for arch in ["all", "amd64", "i386", "arm64"]} +USER_AGENT = "Debian APT-HTTP/1.3 (2.6.1)" # from Debian 12 + """ repo info json format: -{ - "name": repo name +"repo_name": { "repo": repo url, end with "/" - "xxx_path": repo xxx Packages file path, start with no "/" + "xxx_path": { + "arch": repo Packages file path of "arch", start with no "/" + } } """ -def read_repo_list(repo_list_file): +def read_repo_list(repo_list_file: str) -> dict: try: with open(repo_list_file, "r") as f: return json.load(f) except Exception as e: logging.error(f"Error reading repo list: {e}") - return [] + return {} -# get the packages file content from remote repo -def get_remote_packages(repo_url, file_path): +def get_remote_packages(repo_url: str, file_path: str) -> bytes: + """ + get the packages file content from remote repo + """ file_url = repo_url + file_path try: - response = requests.get(file_url, timeout=10) + response = requests.get( + file_url, timeout=10, headers={"User-Agent": USER_AGENT} + ) if response.status_code != 200: logging.error( f"GetError: {file_url} returned status {response.status_code}" @@ -67,9 +74,11 @@ def get_remote_packages(repo_url, file_path): return b"" -def get_latest(deb_packages): - # split the information of each packet, store it in infoList - # 将每个包的信息分割开，存放到 infoList 中 +def get_latest(deb_packages: bytes): + """ + split the information of each packet, deduplication and store the latest in infoList + 将每个包的信息分割开，去重并将最新的存放到 infoList 中 + """ deb_packages = re.sub(rb"^Package: ", b"{{start}}Package: ", deb_packages, flags=re.MULTILINE) info_list = deb_packages.split(b"{{start}}")[1:] @@ -97,18 +106,14 @@ def get_latest(deb_packages): return -def process_repo(r): +def process_repo(r: dict): + """ + 获取仓库中不同架构子仓库的内容，最后调用 get_latest 去重并保存。 + """ try: deb_packages = b"" - if r.get("mix_path"): # 获取扁平 Repo 中包信息 - deb_packages += get_remote_packages(r["repo"], r["mix_path"]) - else: - if r.get("amd64_path"): # 获取 Repo 中 Amd64 包信息 - deb_packages += get_remote_packages(r["repo"], r["amd64_path"]) - if r.get("arm64_path"): # 获取 Repo 中 Arm64 包信息 - deb_packages += get_remote_packages(r["repo"], r["arm64_path"]) - if r.get("all_path"): # 获取 Repo 中 All 包信息 - deb_packages += get_remote_packages(r["repo"], r["all_path"]) + for arch, path in r["path"].items(): + deb_packages += get_remote_packages(r["repo"], path) get_latest(deb_packages) except Exception as e: logging.error(f"Error processing repo {r.get('name', 'unknown')}: {e}") @@ -137,19 +142,19 @@ if __name__ == "__main__": with open(args.local) as f: get_latest(f.read().encode()) - repo_list_file = args.repo - repo_list = read_repo_list(repo_list_file) + # 读取 repo_list 配置 + repo_list = read_repo_list(args.repo) if not repo_list: sys.exit() # 多线程，同时限制最大线程数 with ThreadPoolExecutor(max_workers=10) as executor: - executor.map(process_repo, repo_list) + executor.map(process_repo, repo_list.values()) + # 分别输出到不同文件 os.makedirs("deb/amd64/", exist_ok=True) os.makedirs("deb/arm64/", exist_ok=True) - # 分别输出到不同文件 with open("deb/amd64/Packages", "+wb") as f: for i in package_info["amd64"].values(): f.write(i) diff --git a/run.sh b/run.sh index 52c7013..33ea00b 100755 --- a/run.sh +++ b/run.sh @@ -7,7 +7,7 @@ gen_release() { } # check for updates -$HOME/go/bin/github-downloader -r -o deb +./get-github-releases.py find get -type f -name "*.sh" -exec sh {} \; cd deb