update firefox

rewrite github downloader and configuration to store version tag in sqlite.
add 'all' arch, change 'x86_64' to 'amd64'.
add multithreading for downloader.
rename init_deb.py to init-deb.py

.gitignore

@@ -1,3 +1,4 @@
 data/github-local.json
 data/deb.db
 deb/
+__pycache__/

README.md

@@ -68,7 +68,7 @@
 
 | 软件仓库 | 包名 | amd64 | arm64 |
 | ------ | ---- | ----- | ----- |
-|Mozilla Firefox|firefox<br />firefox_beta<br />firefox_devedition<br />firefox_esr<br />firefox_nightly<br />mozillavpn| ✅ | ✅<br />官方仅提供<br />firefox_nightly |
+|Mozilla Firefox|firefox<br />firefox_beta<br />firefox_devedition<br />firefox_nightly<br />firefox_esr<br />mozillavpn| ✅ | ✅ |
 |Google Chrome|google-chrome-stable<br />google-chrome-beta<br />google-chrome-unstable| ✅ | |
 |Google Earth|google-earth-pro-stable<br />google-earth-ec-stable| ✅ | |
 |Microsoft Edge|microsoft-edge-stable<br />microsoft-edge-beta<br />microsoft-edge-dev| ✅ | |

check_downloader.py

@@ -2,9 +2,16 @@
 import subprocess
 import os
 import sqlite3
+import sys
 import logging
+from threading import Lock
+
+BASE_DIR = "deb"
+DB_DIR = "data"
+USER_AGENT = "Debian APT-HTTP/1.3 (2.6.1)"  # from Debian 12
+
+version_lock = Lock()
 
-base_dir = "deb"
 logging.basicConfig(
     format="%(asctime)s %(message)s",
     datefmt="%Y/%m/%d %H:%M:%S",
@@ -13,68 +20,58 @@ logging.basicConfig(
 
 
 def download(url):
-    file_dir = os.path.join(base_dir, os.path.dirname(url))
-    if not os.path.exists(file_dir):
-        os.makedirs(file_dir)
-    file_path = os.path.join(base_dir, url.split("?")[0])
-    # 用 curl 模拟 apt 下载文件，User-Agent 来自 Debian 12
-    subprocess.run(
-        [
-            "curl",
-            "-H",
-            "User-Agent: Debian APT-HTTP/1.3 (2.6.1)",
-            "-fsLo",
-            file_path,
-            url,
-        ]
-    )
+    """Download file using curl with APT User-Agent."""
+    file_path = os.path.join(BASE_DIR, url.split("?")[0])
+    os.makedirs(os.path.dirname(file_path), exist_ok=True)
+    subprocess.run(["curl", "-H", f"User-Agent: {USER_AGENT}", "-fsLo", file_path, url])
 
 
-def check_download(name, version, url, arch):
+def check_download(name, version, url, arch="amd64"):
+    """Check and handle package download/update."""
     logging.info("%s:%s = %s", name, arch, version)
 
-    # connect to db
-    with sqlite3.connect(os.path.join("data", f"{base_dir}.db")) as conn:
-        cur = conn.cursor()
-        res = cur.execute(
-            f"SELECT version, url FROM {arch} WHERE name = ?", (name,)
-        ).fetchall()
-        if len(res):
-            local_version = res[0][0]
-            local_url = res[0][1]
-            if local_version != version:
-                print(f"Update: {name}:{arch} ({local_version} -> {version})")
-                download(url)
-                # wirte to db
-                cur.execute(
-                    f"UPDATE {arch} SET version = ?, url = ? WHERE name = ?",
+    db_path = os.path.join("data", f"{BASE_DIR}.db")
+    # get local version
+    with version_lock, sqlite3.connect(db_path) as conn:
+        res = conn.execute(
+            f"SELECT version, url FROM '{arch}' WHERE name = ?", (name,)
+        ).fetchone()
+    if res:
+        local_version, local_url = res
+        if local_version != version:
+            print(f"Update: {name}:{arch} ({local_version} -> {version})")
+            download(url)
+            # update database
+            with version_lock, sqlite3.connect(db_path) as conn:
+                conn.execute(
+                    f"UPDATE '{arch}' SET version = ?, url = ? WHERE name = ?",
                     (version, url, name),
                 )
-                # remove old version
-                if local_url != url:  # 针对固定下载链接
-                    old_file_path = os.path.join(base_dir, local_url.split("?")[0])
-                    if os.path.exists(old_file_path):
-                        os.remove(old_file_path)
-        else:
-            print(f"AddNew: {name}:{arch} ({version})")
-            download(url)
-            # wirte to db
-            cur.execute(
-                f"INSERT INTO {arch}(name, version, url) VALUES (?, ?, ?)",
+                conn.commit()
+            # remove old version
+            if local_url != url:  # 防止固定下载链接
+                old_file_path = os.path.join(BASE_DIR, local_url.split("?")[0])
+                if os.path.exists(old_file_path):
+                    os.remove(old_file_path)
+    else:
+        print(f"AddNew: {name}:{arch} ({version})")
+        download(url)
+        # update database
+        with version_lock, sqlite3.connect(db_path) as conn:
+            conn.execute(
+                f"INSERT INTO '{arch}'(name, version, url) VALUES (?, ?, ?)",
                 (name, version, url),
             )
-        conn.commit()
+            conn.commit()
 
 
 if __name__ == "__main__":
-    args = os.sys.argv
-    if len(args) == 5:
-        check_download(args[1], args[2], args[3], args[4])
-    elif len(args) == 4:
-        check_download(args[1], args[2], args[3], "x86_64")
+    args = sys.argv
+    if len(args) in (4, 5):
+        check_download(*args[1:])
     elif len(args) > 1:
         logging.error(f"Unknown Args: {args[1:]}")
     else:
         print(f"Usage: {args[0]} <package_name> <version> <url> [arch]")
         print("options:")
-        print("    arch: x86_64, arm64. default is x86_64")
+        print("    arch: amd64, arm64, all. default is amd64")

data/github.json

@@ -1,108 +1,108 @@
 {
     "clash-verge": {
         "repo": "clash-verge-rev/clash-verge-rev",
-        "file_list": [
-            "Clash.Verge_{stripped_version}_amd64.deb",
-            "Clash.Verge_{stripped_version}_arm64.deb"
-        ]
+        "file_list": {
+            "amd64": "Clash.Verge_{stripped_version}_amd64.deb",
+            "arm64": "Clash.Verge_{stripped_version}_arm64.deb"
+        }
     },
     "mihomo": {
         "repo": "MetaCubeX/mihomo",
-        "file_list": [
-            "mihomo-linux-amd64-compatible-{version_tag}.deb",
-            "mihomo-linux-arm64-{version_tag}.deb"
-        ]
+        "file_list": {
+            "amd64": "mihomo-linux-amd64-compatible-{version_tag}.deb",
+            "arm64": "mihomo-linux-arm64-{version_tag}.deb"
+        }
     },
     "flclash": {
         "repo": "chen08209/FlClash",
-        "file_list": [
-            "FlClash-{stripped_version}-linux-amd64.deb"
-        ]
+        "file_list": {
+            "amd64": "FlClash-{stripped_version}-linux-amd64.deb"
+        }
     },
     "hugo": {
         "repo": "gohugoio/hugo",
-        "file_list": [
-            "hugo_extended_{stripped_version}_linux-amd64.deb",
-            "hugo_extended_{stripped_version}_linux-arm64.deb"
-        ]
+        "file_list": {
+            "amd64": "hugo_extended_{stripped_version}_linux-amd64.deb",
+            "arm64": "hugo_extended_{stripped_version}_linux-arm64.deb"
+        }
     },
     "rustdesk": {
         "repo": "rustdesk/rustdesk",
-        "file_list": [
-            "rustdesk-{version_tag}-x86_64.deb",
-            "rustdesk-{version_tag}-aarch64.deb"
-        ]
+        "file_list": {
+            "amd64": "rustdesk-{version_tag}-x86_64.deb",
+            "arm64": "rustdesk-{version_tag}-aarch64.deb"
+        }
     },
     "obsidian": {
         "repo": "obsidianmd/obsidian-releases",
-        "file_list": [
-            "obsidian_{stripped_version}_amd64.deb"
-        ]
+        "file_list": {
+            "amd64": "obsidian_{stripped_version}_amd64.deb"
+        }
     },
     "tabby": {
         "repo": "Eugeny/tabby",
-        "file_list": [
-            "tabby-{stripped_version}-linux-x64.deb",
-            "tabby-{stripped_version}-linux-arm64.deb"
-        ]
+        "file_list": {
+            "amd64": "tabby-{stripped_version}-linux-x64.deb",
+            "arm64": "tabby-{stripped_version}-linux-arm64.deb"
+        }
     },
     "pandoc": {
         "repo": "jgm/pandoc",
-        "file_list": [
-            "pandoc-{version_tag}-1-amd64.deb",
-            "pandoc-{version_tag}-1-arm64.deb"
-        ]
+        "file_list": {
+            "amd64": "pandoc-{version_tag}-1-amd64.deb",
+            "arm64": "pandoc-{version_tag}-1-arm64.deb"
+        }
     },
     "localsend": {
         "repo": "localsend/localsend",
-        "file_list": [
-            "LocalSend-{stripped_version}-linux-x86-64.deb",
-            "LocalSend-{stripped_version}-linux-arm-64.deb"
-        ]
+        "file_list": {
+            "amd64": "LocalSend-{stripped_version}-linux-x86-64.deb",
+            "arm64": "LocalSend-{stripped_version}-linux-arm-64.deb"
+        }
     },
     "motrix": {
         "repo": "agalwood/Motrix",
-        "file_list": [
-            "Motrix_{stripped_version}_amd64.deb",
-            "Motrix_{stripped_version}_arm64.deb"
-        ]
+        "file_list": {
+            "amd64": "Motrix_{stripped_version}_amd64.deb",
+            "arm64": "Motrix_{stripped_version}_arm64.deb"
+        }
     },
     "peazip": {
         "repo": "peazip/PeaZip",
-        "file_list": [
-            "peazip_{version_tag}.LINUX.GTK2-1_amd64.deb"
-        ]
+        "file_list": {
+            "amd64": "peazip_{version_tag}.LINUX.GTK2-1_amd64.deb"
+        }
     },
     "neovim": {
         "repo": "neovim/neovim-releases",
-        "file_list": [
-            "nvim-linux64.deb"
-        ]
+        "file_list": {
+            "amd64": "nvim-linux-x86_64.deb"
+        }
     },
     "hiddify": {
         "repo": "hiddify/hiddify-app",
-        "file_list": [
-            "Hiddify-Debian-x64.deb"
-        ]
+        "file_list": {
+            "amd64": "Hiddify-Debian-x64.deb"
+        }
     },
     "cloudflared": {
         "repo": "cloudflare/cloudflared",
-        "file_list": [
-            "cloudflared-linux-amd64.deb",
-            "cloudflared-linux-arm64.deb"
-        ]
+        "file_list": {
+            "amd64": "cloudflared-linux-amd64.deb",
+            "arm64": "cloudflared-linux-arm64.deb"
+        }
     },
     "caddy": {
         "repo": "caddyserver/caddy",
-        "file_list": [
-            "caddy_{stripped_version}_linux_amd64.deb",
-            "caddy_{stripped_version}_linux_arm64.deb"
-        ]
+        "file_list": {
+            "amd64": "caddy_{stripped_version}_linux_amd64.deb",
+            "arm64": "caddy_{stripped_version}_linux_arm64.deb"
+        }
     },
     "foliate": {
         "repo": "johnfactotum/foliate",
-        "file_list": [
-            "foliate_{version_tag}_all.deb"
-        ]
+        "file_list": {
+            "all": "foliate_{version_tag}_all.deb"
+        }
     }
 }

get-github-releases.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+import argparse
+import requests
+import json
+import os
+import re
+from concurrent.futures import ThreadPoolExecutor, wait
+from check_downloader import check_download
+
+github_info_list = {}
+
+CONFIG = {"data_dir": "data", "proxy": "", "thread": 5}
+
+
+# 读取命令行参数
+def read_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-d", "--data", default="data", help="从 <DATA> 读取仓库配置")
+    parser.add_argument(
+        "-p", "--proxy", default="", help="Github 代理，<PROXY> 必须以 / 结尾"
+    )
+    parser.add_argument(
+        "-t", "--thread", type=int, default=5, help="并发下载线程数量，默认为 5"
+    )
+    args = parser.parse_args()
+    CONFIG.update({"data_dir": args.data, "proxy": args.proxy, "thread": args.thread})
+
+
+if __name__ == "__main__":
+    read_args()
+
+    # read all repo info 读取所有仓库配置
+    with open(os.path.join(CONFIG["data_dir"], "github.json"), "r") as f:
+        github_info_list = json.load(f)
+
+    tasks = []
+    with ThreadPoolExecutor(max_workers=CONFIG["thread"]) as executor:
+        for name, repo in github_info_list.items():
+            release_url = (
+                f'{CONFIG["proxy"]}https://github.com/{repo["repo"]}/releases'
+            )
+            # get latest version tag 获取最新版本标签
+            location = requests.head(release_url + "/latest").headers.get("Location", "")
+            match = re.search(r".*releases/tag/([^/]+)", location)
+            if not match:
+                continue
+            version_tag = match.group(1)
+            version = version_tag[1:] if version_tag[0].lower() == "v" else version_tag
+
+            for arch, file_name in repo["file_list"].items():
+                release_file = file_name.format(
+                    version_tag=version_tag, stripped_version=version
+                )
+                file_url = f"{release_url}/download/{version_tag}/{release_file}"
+                # 提交任务到线程池
+                tasks.append(
+                    executor.submit(check_download, name, version, file_url, arch)
+                )
+        # 等待所有任务完成
+        wait(tasks)

init-deb.py

@@ -6,7 +6,7 @@ import sqlite3
 conn = sqlite3.connect("data/deb.db")
 conn.execute(
     """
-        CREATE TABLE IF NOT EXISTS x86_64 (
+        CREATE TABLE IF NOT EXISTS amd64 (
             name TEXT UNIQUE,
             version TEXT,
             url TEXT
@@ -22,6 +22,15 @@ conn.execute(
         );
     """
 )
+conn.execute(
+    """
+        CREATE TABLE IF NOT EXISTS 'all' (
+            name TEXT UNIQUE,
+            version TEXT,
+            url TEXT
+        );
+    """
+)
 
 conn.commit()
 conn.close()

run.sh

@@ -7,7 +7,7 @@ gen_release() {
 }
 
 # check for updates
-$HOME/go/bin/github-downloader -r -o deb
+./get-github-releases.py
 find get -type f -name "*.sh" -exec sh {} \;
 
 cd deb