diff --git a/data/repo_list.json b/data/repo_list.json index 1e6b961..40f0148 100644 --- a/data/repo_list.json +++ b/data/repo_list.json @@ -2,17 +2,17 @@ { "name": "google-chrome", "repo": "https://dl.google.com/linux/chrome/deb/", - "amd64_path": "dists/stable/main/binary-amd64/Packages" + "amd64_path": "dists/stable/main/binary-amd64/Packages.gz" }, { "name": "termius", "repo": "https://deb.termius.com/", - "amd64_path": "dists/squeeze/main/binary-amd64/Packages" + "amd64_path": "dists/squeeze/main/binary-amd64/Packages.gz" }, { "name": "steam", "repo": "https://repo.steampowered.com/steam/", - "amd64_path": "dists/stable/steam/binary-amd64/Packages" + "amd64_path": "dists/stable/steam/binary-amd64/Packages.gz" }, { "name": "firefox", @@ -23,24 +23,24 @@ { "name": "microsoft-edge", "repo": "https://packages.microsoft.com/repos/edge/", - "amd64_path": "dists/stable/main/binary-amd64/Packages" + "amd64_path": "dists/stable/main/binary-amd64/Packages.gz" }, { "name": "code", "repo": "https://packages.microsoft.com/repos/code/", - "amd64_path": "dists/stable/main/binary-amd64/Packages", - "arm64_path": "dists/stable/main/binary-arm64/Packages" + "amd64_path": "dists/stable/main/binary-amd64/Packages.gz", + "arm64_path": "dists/stable/main/binary-arm64/Packages.gz" }, { "name": "tailscale", "repo": "https://pkgs.tailscale.com/stable/debian/", - "amd64_path": "dists/sid/main/binary-amd64/Packages", - "arm64_path": "dists/sid/main/binary-arm64/Packages" + "amd64_path": "dists/sid/main/binary-amd64/Packages.gz", + "arm64_path": "dists/sid/main/binary-arm64/Packages.gz" }, { "name": "sublime", "repo": "https://download.sublimetext.com/", - "mix_path": "apt/stable/Packages" + "mix_path": "apt/stable/Packages.gz" }, { "name": "black-desk", @@ -50,7 +50,7 @@ { "name": "typora", "repo": "https://typoraio.cn/linux/", - "mix_path": "Packages" + "mix_path": "Packages.gz" }, { "name": "zotero", @@ -60,8 +60,8 @@ { "name": "gh", "repo": "https://cli.github.com/packages/", - "amd64_path": "dists/stable/main/binary-amd64/Packages", - "arm64_path": "dists/stable/main/binary-arm64/Packages" + "amd64_path": "dists/stable/main/binary-amd64/Packages.gz", + "arm64_path": "dists/stable/main/binary-arm64/Packages.gz" }, { "name": "ttyd", diff --git a/init_deb.py b/init_deb.py index 46d6288..1ac0353 100755 --- a/init_deb.py +++ b/init_deb.py @@ -1,14 +1,7 @@ #!/usr/bin/env python3 -import os import sqlite3 -# create dir -if not os.path.exists("deb/amd64"): - os.makedirs("deb/amd64") -if not os.path.exists("deb/arm64"): - os.makedirs("deb/arm64") - # create table conn = sqlite3.connect("data/deb.db") conn.execute( diff --git a/merge-apt-repo.py b/merge-apt-repo.py index b9bf8b7..dcf42e0 100755 --- a/merge-apt-repo.py +++ b/merge-apt-repo.py @@ -1,8 +1,11 @@ #!/usr/bin/env python3 import argparse +import gzip +import io import json import logging +import lzma import os import re import requests @@ -10,16 +13,14 @@ import sys from concurrent.futures import ThreadPoolExecutor from threading import Lock -config = {} -package_version = {"all": {}, "amd64": {}, "i386": {}, "arm64": {}} -package_info = {"all": {}, "amd64": {}, "i386": {}, "arm64": {}} -lock = {"all": Lock(), "amd64": Lock(), "i386": Lock(), "arm64": Lock()} +package_version = {arch: {} for arch in ["all", "amd64", "i386", "arm64"]} +package_info = {arch: {} for arch in ["all", "amd64", "i386", "arm64"]} +lock = {arch: Lock() for arch in ["all", "amd64", "i386", "arm64"]} """ repo info json format: { "name": repo name - "only_latest": only has the latest version or not "repo": repo url, end with "/" "xxx_path": repo xxx Packages file path, start with no "/" } @@ -45,7 +46,17 @@ def get_remote_packages(repo_url, file_path): f"GetError: {file_url} returned status {response.status_code}" ) return b"" - content = response.content + + content = b"" + if file_url.endswith(".gz"): # Packages.gz + with gzip.GzipFile(fileobj=io.BytesIO(response.content)) as f: + content = f.read() + elif file_url.endswith(".xz"): # Packages.xz + with lzma.LZMAFile(io.BytesIO(response.content)) as f: + content = f.read() + else: # Packages + content = response.content + # complete the two newlines if the ending is less than two newlines # 结尾不足两个换行符的话,补全两个换行符 if not content.endswith(b"\n\n"): @@ -123,26 +134,27 @@ if __name__ == "__main__": with open(args.local) as f: get_latest(f.read().encode()) - repo_list = args.repo - repo_list = read_repo_list(repo_list) + repo_list_file = args.repo + repo_list = read_repo_list(repo_list_file) if not repo_list: sys.exit() + # 多线程,同时限制最大线程数 with ThreadPoolExecutor(max_workers=10) as executor: executor.map(process_repo, repo_list) + os.makedirs("deb/amd64/", exist_ok=True) + os.makedirs("deb/arm64/", exist_ok=True) + # 分别输出到不同文件 with open("deb/amd64/Packages", "+wb") as f: for i in package_info["amd64"].values(): f.write(i) - for i in package_info["i386"].values(): - f.write(i) for i in package_info["all"].values(): f.write(i) - f.close + with open("deb/arm64/Packages", "+wb") as f: for i in package_info["arm64"].values(): f.write(i) for i in package_info["all"].values(): f.write(i) - f.close