feat: merge with Packages.gz and Packages.xz

data/repo_list.json

@@ -2,17 +2,17 @@
     {
         "name": "google-chrome",
         "repo": "https://dl.google.com/linux/chrome/deb/",
-        "amd64_path": "dists/stable/main/binary-amd64/Packages"
+        "amd64_path": "dists/stable/main/binary-amd64/Packages.gz"
     },
     {
         "name": "termius",
         "repo": "https://deb.termius.com/",
-        "amd64_path": "dists/squeeze/main/binary-amd64/Packages"
+        "amd64_path": "dists/squeeze/main/binary-amd64/Packages.gz"
     },
     {
         "name": "steam",
         "repo": "https://repo.steampowered.com/steam/",
-        "amd64_path": "dists/stable/steam/binary-amd64/Packages"
+        "amd64_path": "dists/stable/steam/binary-amd64/Packages.gz"
     },
     {
         "name": "firefox",
@@ -23,24 +23,24 @@
     {
         "name": "microsoft-edge",
         "repo": "https://packages.microsoft.com/repos/edge/",
-        "amd64_path": "dists/stable/main/binary-amd64/Packages"
+        "amd64_path": "dists/stable/main/binary-amd64/Packages.gz"
     },
     {
         "name": "code",
         "repo": "https://packages.microsoft.com/repos/code/",
-        "amd64_path": "dists/stable/main/binary-amd64/Packages",
+        "amd64_path": "dists/stable/main/binary-amd64/Packages.gz",
-        "arm64_path": "dists/stable/main/binary-arm64/Packages"
+        "arm64_path": "dists/stable/main/binary-arm64/Packages.gz"
     },
     {
         "name": "tailscale",
         "repo": "https://pkgs.tailscale.com/stable/debian/",
-        "amd64_path": "dists/sid/main/binary-amd64/Packages",
+        "amd64_path": "dists/sid/main/binary-amd64/Packages.gz",
-        "arm64_path": "dists/sid/main/binary-arm64/Packages"
+        "arm64_path": "dists/sid/main/binary-arm64/Packages.gz"
     },
     {
         "name": "sublime",
         "repo": "https://download.sublimetext.com/",
-        "mix_path": "apt/stable/Packages"
+        "mix_path": "apt/stable/Packages.gz"
     },
     {
         "name": "black-desk",
@@ -50,7 +50,7 @@
     {
         "name": "typora",
         "repo": "https://typoraio.cn/linux/",
-        "mix_path": "Packages"
+        "mix_path": "Packages.gz"
     },
     {
         "name": "zotero",
@@ -60,8 +60,8 @@
     {
         "name": "gh",
         "repo": "https://cli.github.com/packages/",
-        "amd64_path": "dists/stable/main/binary-amd64/Packages",
+        "amd64_path": "dists/stable/main/binary-amd64/Packages.gz",
-        "arm64_path": "dists/stable/main/binary-arm64/Packages"
+        "arm64_path": "dists/stable/main/binary-arm64/Packages.gz"
     },
     {
         "name": "ttyd",

init_deb.py

@@ -1,14 +1,7 @@
 #!/usr/bin/env python3
 
-import os
 import sqlite3
 
-# create dir
-if not os.path.exists("deb/amd64"):
-    os.makedirs("deb/amd64")
-if not os.path.exists("deb/arm64"):
-    os.makedirs("deb/arm64")
-
 # create table
 conn = sqlite3.connect("data/deb.db")
 conn.execute(

merge-apt-repo.py

@@ -1,8 +1,11 @@
 #!/usr/bin/env python3
 
 import argparse
+import gzip
+import io
 import json
 import logging
+import lzma
 import os
 import re
 import requests
@@ -10,16 +13,14 @@ import sys
 from concurrent.futures import ThreadPoolExecutor
 from threading import Lock
 
-config = {}
+package_version = {arch: {} for arch in ["all", "amd64", "i386", "arm64"]}
-package_version = {"all": {}, "amd64": {}, "i386": {}, "arm64": {}}
+package_info = {arch: {} for arch in ["all", "amd64", "i386", "arm64"]}
-package_info = {"all": {}, "amd64": {}, "i386": {}, "arm64": {}}
+lock = {arch: Lock() for arch in ["all", "amd64", "i386", "arm64"]}
-lock = {"all": Lock(), "amd64": Lock(), "i386": Lock(), "arm64": Lock()}
 
 """
 repo info json format:
 {
     "name": repo name
-    "only_latest": only has the latest version or not
     "repo": repo url, end with "/"
     "xxx_path": repo xxx Packages file path, start with no "/"
 }
@@ -45,7 +46,17 @@ def get_remote_packages(repo_url, file_path):
                 f"GetError: {file_url} returned status {response.status_code}"
             )
             return b""
-        content = response.content
+
+        content = b""
+        if file_url.endswith(".gz"):  # Packages.gz
+            with gzip.GzipFile(fileobj=io.BytesIO(response.content)) as f:
+                content = f.read()
+        elif file_url.endswith(".xz"):  # Packages.xz
+            with lzma.LZMAFile(io.BytesIO(response.content)) as f:
+                content = f.read()
+        else:  # Packages
+            content = response.content
+
         # complete the two newlines if the ending is less than two newlines
         # 结尾不足两个换行符的话，补全两个换行符
         if not content.endswith(b"\n\n"):
@@ -123,26 +134,27 @@ if __name__ == "__main__":
         with open(args.local) as f:
             get_latest(f.read().encode())
 
-    repo_list = args.repo
+    repo_list_file = args.repo
-    repo_list = read_repo_list(repo_list)
+    repo_list = read_repo_list(repo_list_file)
     if not repo_list:
         sys.exit()
+
     # 多线程，同时限制最大线程数
     with ThreadPoolExecutor(max_workers=10) as executor:
         executor.map(process_repo, repo_list)
 
+    os.makedirs("deb/amd64/", exist_ok=True)
+    os.makedirs("deb/arm64/", exist_ok=True)
+
     # 分别输出到不同文件
     with open("deb/amd64/Packages", "+wb") as f:
         for i in package_info["amd64"].values():
             f.write(i)
-        for i in package_info["i386"].values():
-            f.write(i)
         for i in package_info["all"].values():
             f.write(i)
-        f.close
+
     with open("deb/arm64/Packages", "+wb") as f:
         for i in package_info["arm64"].values():
             f.write(i)
         for i in package_info["all"].values():
             f.write(i)
-        f.close