update firefox

rewrite github downloader and configuration to store version tag in sqlite.
add 'all' arch, change 'x86_64' to 'amd64'.
add multithreading for downloader.
rename init_deb.py to init-deb.py

data/github.json

@@ -2,75 +2,75 @@
     "clash-verge": {
         "repo": "clash-verge-rev/clash-verge-rev",
         "file_list": {
-            "amd64": "Clash.Verge_{version}_amd64.deb",
-            "arm64": "Clash.Verge_{version}_arm64.deb"
+            "amd64": "Clash.Verge_{stripped_version}_amd64.deb",
+            "arm64": "Clash.Verge_{stripped_version}_arm64.deb"
         }
     },
     "mihomo": {
         "repo": "MetaCubeX/mihomo",
         "file_list": {
-            "amd64": "mihomo-linux-amd64-compatible-{releases_tag}.deb",
-            "arm64": "mihomo-linux-arm64-{releases_tag}.deb"
+            "amd64": "mihomo-linux-amd64-compatible-{version_tag}.deb",
+            "arm64": "mihomo-linux-arm64-{version_tag}.deb"
         }
     },
     "flclash": {
         "repo": "chen08209/FlClash",
         "file_list": {
-            "amd64": "FlClash-{version}-linux-amd64.deb"
+            "amd64": "FlClash-{stripped_version}-linux-amd64.deb"
         }
     },
     "hugo": {
         "repo": "gohugoio/hugo",
         "file_list": {
-            "amd64": "hugo_extended_{version}_linux-amd64.deb",
-            "arm64": "hugo_extended_{version}_linux-arm64.deb"
+            "amd64": "hugo_extended_{stripped_version}_linux-amd64.deb",
+            "arm64": "hugo_extended_{stripped_version}_linux-arm64.deb"
         }
     },
     "rustdesk": {
         "repo": "rustdesk/rustdesk",
         "file_list": {
-            "amd64": "rustdesk-{releases_tag}-x86_64.deb",
-            "arm64": "rustdesk-{releases_tag}-aarch64.deb"
+            "amd64": "rustdesk-{version_tag}-x86_64.deb",
+            "arm64": "rustdesk-{version_tag}-aarch64.deb"
         }
     },
     "obsidian": {
         "repo": "obsidianmd/obsidian-releases",
         "file_list": {
-            "amd64": "obsidian_{version}_amd64.deb"
+            "amd64": "obsidian_{stripped_version}_amd64.deb"
         }
     },
     "tabby": {
         "repo": "Eugeny/tabby",
         "file_list": {
-            "amd64": "tabby-{version}-linux-x64.deb",
-            "arm64": "tabby-{version}-linux-arm64.deb"
+            "amd64": "tabby-{stripped_version}-linux-x64.deb",
+            "arm64": "tabby-{stripped_version}-linux-arm64.deb"
         }
     },
     "pandoc": {
         "repo": "jgm/pandoc",
         "file_list": {
-            "amd64": "pandoc-{releases_tag}-1-amd64.deb",
-            "arm64": "pandoc-{releases_tag}-1-arm64.deb"
+            "amd64": "pandoc-{version_tag}-1-amd64.deb",
+            "arm64": "pandoc-{version_tag}-1-arm64.deb"
         }
     },
     "localsend": {
         "repo": "localsend/localsend",
         "file_list": {
-            "amd64": "LocalSend-{version}-linux-x86-64.deb",
-            "arm64": "LocalSend-{version}-linux-arm-64.deb"
+            "amd64": "LocalSend-{stripped_version}-linux-x86-64.deb",
+            "arm64": "LocalSend-{stripped_version}-linux-arm-64.deb"
         }
     },
     "motrix": {
         "repo": "agalwood/Motrix",
         "file_list": {
-            "amd64": "Motrix_{version}_amd64.deb",
-            "arm64": "Motrix_{version}_arm64.deb"
+            "amd64": "Motrix_{stripped_version}_amd64.deb",
+            "arm64": "Motrix_{stripped_version}_arm64.deb"
         }
     },
     "peazip": {
         "repo": "peazip/PeaZip",
         "file_list": {
-            "amd64": "peazip_{releases_tag}.LINUX.GTK2-1_amd64.deb"
+            "amd64": "peazip_{version_tag}.LINUX.GTK2-1_amd64.deb"
         }
     },
     "neovim": {
@@ -95,14 +95,14 @@
     "caddy": {
         "repo": "caddyserver/caddy",
         "file_list": {
-            "amd64": "caddy_{version}_linux_amd64.deb",
-            "arm64": "caddy_{version}_linux_arm64.deb"
+            "amd64": "caddy_{stripped_version}_linux_amd64.deb",
+            "arm64": "caddy_{stripped_version}_linux_arm64.deb"
         }
     },
     "foliate": {
         "repo": "johnfactotum/foliate",
         "file_list": {
-            "all": "foliate_{releases_tag}_all.deb"
+            "all": "foliate_{version_tag}_all.deb"
         }
     }
 }

data/repo_list.json

@@ -1,126 +1,106 @@
+[
     {
-    "google-chrome": {
+        "name": "google-chrome",
         "repo": "https://dl.google.com/linux/chrome/deb/",
-        "path": {
-            "amd64": "dists/stable/main/binary-amd64/Packages.gz"
-        }
+        "amd64_path": "dists/stable/main/binary-amd64/Packages.gz"
     },
-    "google-earth": {
+    {
+        "name": "google-earth",
         "repo": "https://dl.google.com/linux/earth/deb/",
-        "path": {
-            "amd64": "dists/stable/main/binary-amd64/Packages.gz"
-        }
+        "amd64_path": "dists/stable/main/binary-amd64/Packages.gz"
     },
-    "termius": {
+    {
+        "name": "termius",
         "repo": "https://deb.termius.com/",
-        "path": {
-            "amd64": "dists/squeeze/main/binary-amd64/Packages.gz"
-        }
+        "amd64_path": "dists/squeeze/main/binary-amd64/Packages.gz"
     },
-    "steam": {
+    {
+        "name": "steam",
         "repo": "https://repo.steampowered.com/steam/",
-        "path": {
-            "amd64": "dists/stable/steam/binary-amd64/Packages.gz"
-        }
+        "amd64_path": "dists/stable/steam/binary-amd64/Packages.gz"
     },
-    "firefox": {
+    {
+        "name": "firefox",
         "repo": "https://packages.mozilla.org/apt/",
-        "path": {
-            "amd64": "dists/mozilla/main/binary-amd64/Packages",
-            "arm64": "dists/mozilla/main/binary-arm64/Packages"
-        }
+        "amd64_path": "dists/mozilla/main/binary-amd64/Packages",
+        "arm64_path": "dists/mozilla/main/binary-arm64/Packages"
     },
-    "microsoft-edge": {
+    {
+        "name": "microsoft-edge",
         "repo": "https://packages.microsoft.com/repos/edge/",
-        "path": {
-            "amd64": "dists/stable/main/binary-amd64/Packages.gz"
-        }
+        "amd64_path": "dists/stable/main/binary-amd64/Packages.gz"
     },
-    "opera-stable": {
+    {
+        "name": "opera-stable",
         "repo": "https://deb.opera.com/opera-stable/",
-        "path": {
-            "amd64": "dists/stable/non-free/binary-amd64/Packages.gz"
-        }
+        "amd64_path": "dists/stable/non-free/binary-amd64/Packages.gz"
     },
-    "code": {
+    {
+        "name": "code",
         "repo": "https://packages.microsoft.com/repos/code/",
-        "path": {
-            "amd64": "dists/stable/main/binary-amd64/Packages.gz",
-            "arm64": "dists/stable/main/binary-arm64/Packages.gz"
-        }
+        "amd64_path": "dists/stable/main/binary-amd64/Packages.gz",
+        "arm64_path": "dists/stable/main/binary-arm64/Packages.gz"
     },
-    "tailscale": {
+    {
+        "name": "tailscale",
         "repo": "https://pkgs.tailscale.com/stable/debian/",
-        "path": {
-            "amd64": "dists/sid/main/binary-amd64/Packages.gz",
-            "arm64": "dists/sid/main/binary-arm64/Packages.gz"
-        }
+        "amd64_path": "dists/sid/main/binary-amd64/Packages.gz",
+        "arm64_path": "dists/sid/main/binary-arm64/Packages.gz"
     },
-    "sublime": {
+    {
+        "name": "sublime",
         "repo": "https://download.sublimetext.com/",
-        "path": {
-            "mix": "apt/stable/Packages.gz"
-        }
+        "mix_path": "apt/stable/Packages.gz"
     },
-    "black-desk": {
+    {
+        "name": "black-desk",
         "repo": "https://github.com/black-desk/debs/releases/latest/download/",
-        "path": {
-            "amd64": "Packages"
-        }
+        "amd64_path": "Packages"
     },
-    "typora": {
+    {
+        "name": "typora",
         "repo": "https://typoraio.cn/linux/",
-        "path": {
-            "mix": "Packages.gz"
-        }
+        "mix_path": "Packages.gz"
     },
-    "zotero": {
+    {
+        "name": "zotero",
         "repo": "https://zotero.retorque.re/file/apt-package-archive/",
-        "path": {
-            "amd64": "Packages"
-        }
+        "amd64_path": "Packages"
     },
-    "gh": {
+    {
+        "name": "gh",
         "repo": "https://cli.github.com/packages/",
-        "path": {
-            "amd64": "dists/stable/main/binary-amd64/Packages.gz",
-            "arm64": "dists/stable/main/binary-arm64/Packages.gz"
-        }
+        "amd64_path": "dists/stable/main/binary-amd64/Packages.gz",
+        "arm64_path": "dists/stable/main/binary-arm64/Packages.gz"
     },
-    "dufs": {
+    {
+        "name": "dufs",
         "repo": "https://github.com/wcbing-build/dufs-debs/releases/latest/download/",
-        "path": {
-            "mix": "Packages"
-        }
+        "mix_path": "Packages"
     },
-    "frp": {
+    {
+        "name": "frp",
         "repo": "https://github.com/wcbing-build/frp-debs/releases/latest/download/",
-        "path": {
-            "mix": "Packages"
-        }
+        "mix_path": "Packages"
     },
-    "lazydocker": {
+    {
+        "name": "lazydocker",
         "repo": "https://github.com/wcbing-build/lazydocker-debs/releases/latest/download/",
-        "path": {
-            "mix": "Packages"
-        }
+        "mix_path": "Packages"
     },
-    "lazygit": {
+    {
+        "name": "lazygit",
         "repo": "https://github.com/wcbing-build/lazygit-debs/releases/latest/download/",
-        "path": {
-            "mix": "Packages"
-        }
+        "mix_path": "Packages"
     },
-    "nexttrace": {
+    {
+        "name": "nexttrace",
         "repo": "https://github.com/nxtrace/nexttrace-debs/releases/latest/download/",
-        "path": {
-            "mix": "Packages"
-        }
+        "mix_path": "Packages"
     },
-    "debiancn": {
+    {
+        "name": "debiancn",
         "repo": "https://mirrors.cernet.edu.cn/debiancn/",
-        "path": {
-            "amd64": "dists/bookworm/main/binary-amd64/Packages.gz"
-        }
-    }
+        "amd64_path": "dists/bookworm/main/binary-amd64/Packages.gz"
     }
+]

get-github-releases.py

@@ -39,20 +39,19 @@ if __name__ == "__main__":
             release_url = (
                 f'{CONFIG["proxy"]}https://github.com/{repo["repo"]}/releases'
             )
-            # get latest releases tag 获取最新版本标签
+            # get latest version tag 获取最新版本标签
             location = requests.head(release_url + "/latest").headers.get("Location", "")
             match = re.search(r".*releases/tag/([^/]+)", location)
             if not match:
                 continue
-            releases_tag = match.group(1)
-            version = match.group() if (match := re.search("[0-9].*", releases_tag)) else ""
-
+            version_tag = match.group(1)
+            version = version_tag[1:] if version_tag[0].lower() == "v" else version_tag
 
             for arch, file_name in repo["file_list"].items():
                 release_file = file_name.format(
-                    releases_tag=releases_tag, version=version
+                    version_tag=version_tag, stripped_version=version
                 )
-                file_url = f"{release_url}/download/{releases_tag}/{release_file}"
+                file_url = f"{release_url}/download/{version_tag}/{release_file}"
                 # 提交任务到线程池
                 tasks.append(
                     executor.submit(check_download, name, version, file_url, arch)

init-deb.py

@@ -4,10 +4,27 @@ import sqlite3
 
 # create table
 conn = sqlite3.connect("data/deb.db")
-for arch in ["amd64", "arm64", "all"]:
 conn.execute(
-        f"""
-            CREATE TABLE IF NOT EXISTS '{arch}' (
+    """
+        CREATE TABLE IF NOT EXISTS amd64 (
+            name TEXT UNIQUE,
+            version TEXT,
+            url TEXT
+        );
+    """
+)
+conn.execute(
+    """
+        CREATE TABLE IF NOT EXISTS arm64 (
+            name TEXT UNIQUE,
+            version TEXT,
+            url TEXT
+        );
+    """
+)
+conn.execute(
+    """
+        CREATE TABLE IF NOT EXISTS 'all' (
             name TEXT UNIQUE,
             version TEXT,
             url TEXT

merge-apt-repo.py

@@ -17,37 +17,30 @@ package_version = {arch: {} for arch in ["all", "amd64", "i386", "arm64"]}
 package_info = {arch: {} for arch in ["all", "amd64", "i386", "arm64"]}
 lock = {arch: Lock() for arch in ["all", "amd64", "i386", "arm64"]}
 
-USER_AGENT = "Debian APT-HTTP/1.3 (2.6.1)"  # from Debian 12
-
 """
 repo info json format:
-"repo_name": {     
+{
+    "name": repo name
     "repo": repo url, end with "/"
-    "xxx_path": {
-        "arch": repo Packages file path of "arch", start with no "/"
-    }
+    "xxx_path": repo xxx Packages file path, start with no "/"
 }
 """
 
 
-def read_repo_list(repo_list_file: str) -> dict:
+def read_repo_list(repo_list_file):
     try:
         with open(repo_list_file, "r") as f:
             return json.load(f)
     except Exception as e:
         logging.error(f"Error reading repo list: {e}")
-        return {}
+        return []
 
 
-def get_remote_packages(repo_url: str, file_path: str) -> bytes:
-    """
-    get the packages file content from remote repo
-    """
+# get the packages file content from remote repo
+def get_remote_packages(repo_url, file_path):
     file_url = repo_url + file_path
     try:
-        response = requests.get(
-            file_url, timeout=10, headers={"User-Agent": USER_AGENT}
-        )
+        response = requests.get(file_url, timeout=10)
         if response.status_code != 200:
             logging.error(
                 f"GetError: {file_url} returned status {response.status_code}"
@@ -74,11 +67,9 @@ def get_remote_packages(repo_url: str, file_path: str) -> bytes:
         return b""
 
 
-def get_latest(deb_packages: bytes):
-    """
-    split the information of each packet, deduplication and store the latest in infoList
-    将每个包的信息分割开，去重并将最新的存放到 infoList 中
-    """
+def get_latest(deb_packages):
+    # split the information of each packet, store it in infoList
+    # 将每个包的信息分割开，存放到 infoList 中
     deb_packages = re.sub(rb"^Package: ", b"{{start}}Package: ", deb_packages, flags=re.MULTILINE)
     info_list = deb_packages.split(b"{{start}}")[1:]
 
@@ -106,14 +97,18 @@ def get_latest(deb_packages: bytes):
     return
 
 
-def process_repo(r: dict):
-    """
-    获取仓库中不同架构子仓库的内容，最后调用 get_latest 去重并保存。
-    """
+def process_repo(r):
     try:
         deb_packages = b""
-        for arch, path in r["path"].items():
-            deb_packages += get_remote_packages(r["repo"], path)
+        if r.get("mix_path"):  # 获取扁平 Repo 中包信息
+            deb_packages += get_remote_packages(r["repo"], r["mix_path"])
+        else:
+            if r.get("amd64_path"):  # 获取 Repo 中 Amd64 包信息
+                deb_packages += get_remote_packages(r["repo"], r["amd64_path"])
+            if r.get("arm64_path"):  # 获取 Repo 中 Arm64 包信息
+                deb_packages += get_remote_packages(r["repo"], r["arm64_path"])
+            if r.get("all_path"):  # 获取 Repo 中 All 包信息
+                deb_packages += get_remote_packages(r["repo"], r["all_path"])
         get_latest(deb_packages)
     except Exception as e:
         logging.error(f"Error processing repo {r.get('name', 'unknown')}: {e}")
@@ -142,19 +137,19 @@ if __name__ == "__main__":
         with open(args.local) as f:
             get_latest(f.read().encode())
 
-    # 读取 repo_list 配置
-    repo_list = read_repo_list(args.repo)
+    repo_list_file = args.repo
+    repo_list = read_repo_list(repo_list_file)
     if not repo_list:
         sys.exit()
 
     # 多线程，同时限制最大线程数
     with ThreadPoolExecutor(max_workers=10) as executor:
-        executor.map(process_repo, repo_list.values())
+        executor.map(process_repo, repo_list)
 
-    # 分别输出到不同文件
     os.makedirs("deb/amd64/", exist_ok=True)
     os.makedirs("deb/arm64/", exist_ok=True)
 
+    # 分别输出到不同文件
     with open("deb/amd64/Packages", "+wb") as f:
         for i in package_info["amd64"].values():
             f.write(i)