mirror of
https://github.com/wcbing/wcbing-apt-repo.git
synced 2025-12-29 02:28:30 +08:00
refactor: github releases, arch name, multithreading
rewrite github downloader and configuration to store version tag in sqlite. change 'version_tag' to 'releases_tag', 'stripped_version' to 'version'. add 'all' arch, change 'x86_64' to 'amd64'. add multithreading for downloader. rename init_deb.py to init-deb.py
This commit is contained in:
parent
07d25c28c7
commit
2643348df9
3
.gitignore
vendored
3
.gitignore
vendored
@ -1,3 +1,4 @@
|
||||
data/github-local.json
|
||||
data/deb.db
|
||||
deb/
|
||||
deb/
|
||||
__pycache__/
|
||||
@ -2,9 +2,16 @@
|
||||
import subprocess
|
||||
import os
|
||||
import sqlite3
|
||||
import sys
|
||||
import logging
|
||||
from threading import Lock
|
||||
|
||||
BASE_DIR = "deb"
|
||||
DB_DIR = "data"
|
||||
USER_AGENT = "Debian APT-HTTP/1.3 (2.6.1)" # from Debian 12
|
||||
|
||||
version_lock = Lock()
|
||||
|
||||
base_dir = "deb"
|
||||
logging.basicConfig(
|
||||
format="%(asctime)s %(message)s",
|
||||
datefmt="%Y/%m/%d %H:%M:%S",
|
||||
@ -13,68 +20,58 @@ logging.basicConfig(
|
||||
|
||||
|
||||
def download(url):
|
||||
file_dir = os.path.join(base_dir, os.path.dirname(url))
|
||||
if not os.path.exists(file_dir):
|
||||
os.makedirs(file_dir)
|
||||
file_path = os.path.join(base_dir, url.split("?")[0])
|
||||
# 用 curl 模拟 apt 下载文件，User-Agent 来自 Debian 12
|
||||
subprocess.run(
|
||||
[
|
||||
"curl",
|
||||
"-H",
|
||||
"User-Agent: Debian APT-HTTP/1.3 (2.6.1)",
|
||||
"-fsLo",
|
||||
file_path,
|
||||
url,
|
||||
]
|
||||
)
|
||||
"""Download file using curl with APT User-Agent."""
|
||||
file_path = os.path.join(BASE_DIR, url.split("?")[0])
|
||||
os.makedirs(os.path.dirname(file_path), exist_ok=True)
|
||||
subprocess.run(["curl", "-H", f"User-Agent: {USER_AGENT}", "-fsLo", file_path, url])
|
||||
|
||||
|
||||
def check_download(name, version, url, arch):
|
||||
def check_download(name, version, url, arch="amd64"):
|
||||
"""Check and handle package download/update."""
|
||||
logging.info("%s:%s = %s", name, arch, version)
|
||||
|
||||
# connect to db
|
||||
with sqlite3.connect(os.path.join("data", f"{base_dir}.db")) as conn:
|
||||
cur = conn.cursor()
|
||||
res = cur.execute(
|
||||
f"SELECT version, url FROM {arch} WHERE name = ?", (name,)
|
||||
).fetchall()
|
||||
if len(res):
|
||||
local_version = res[0][0]
|
||||
local_url = res[0][1]
|
||||
if local_version != version:
|
||||
print(f"Update: {name}:{arch} ({local_version} -> {version})")
|
||||
download(url)
|
||||
# wirte to db
|
||||
cur.execute(
|
||||
f"UPDATE {arch} SET version = ?, url = ? WHERE name = ?",
|
||||
db_path = os.path.join("data", f"{BASE_DIR}.db")
|
||||
# get local version
|
||||
with version_lock, sqlite3.connect(db_path) as conn:
|
||||
res = conn.execute(
|
||||
f"SELECT version, url FROM '{arch}' WHERE name = ?", (name,)
|
||||
).fetchone()
|
||||
if res:
|
||||
local_version, local_url = res
|
||||
if local_version != version:
|
||||
print(f"Update: {name}:{arch} ({local_version} -> {version})")
|
||||
download(url)
|
||||
# update database
|
||||
with version_lock, sqlite3.connect(db_path) as conn:
|
||||
conn.execute(
|
||||
f"UPDATE '{arch}' SET version = ?, url = ? WHERE name = ?",
|
||||
(version, url, name),
|
||||
)
|
||||
# remove old version
|
||||
if local_url != url: # 针对固定下载链接
|
||||
old_file_path = os.path.join(base_dir, local_url.split("?")[0])
|
||||
if os.path.exists(old_file_path):
|
||||
os.remove(old_file_path)
|
||||
else:
|
||||
print(f"AddNew: {name}:{arch} ({version})")
|
||||
download(url)
|
||||
# wirte to db
|
||||
cur.execute(
|
||||
f"INSERT INTO {arch}(name, version, url) VALUES (?, ?, ?)",
|
||||
conn.commit()
|
||||
# remove old version
|
||||
if local_url != url: # 防止固定下载链接
|
||||
old_file_path = os.path.join(BASE_DIR, local_url.split("?")[0])
|
||||
if os.path.exists(old_file_path):
|
||||
os.remove(old_file_path)
|
||||
else:
|
||||
print(f"AddNew: {name}:{arch} ({version})")
|
||||
download(url)
|
||||
# update database
|
||||
with version_lock, sqlite3.connect(db_path) as conn:
|
||||
conn.execute(
|
||||
f"INSERT INTO '{arch}'(name, version, url) VALUES (?, ?, ?)",
|
||||
(name, version, url),
|
||||
)
|
||||
conn.commit()
|
||||
conn.commit()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = os.sys.argv
|
||||
if len(args) == 5:
|
||||
check_download(args[1], args[2], args[3], args[4])
|
||||
elif len(args) == 4:
|
||||
check_download(args[1], args[2], args[3], "x86_64")
|
||||
args = sys.argv
|
||||
if len(args) in (4, 5):
|
||||
check_download(*args[1:])
|
||||
elif len(args) > 1:
|
||||
logging.error(f"Unknown Args: {args[1:]}")
|
||||
else:
|
||||
print(f"Usage: {args[0]} <package_name> <version> <url> [arch]")
|
||||
print("options:")
|
||||
print(" arch: x86_64, arm64. default is x86_64")
|
||||
print(" arch: amd64, arm64, all. default is amd64")
|
||||
|
||||
116
data/github.json
116
data/github.json
@ -1,108 +1,108 @@
|
||||
{
|
||||
"clash-verge": {
|
||||
"repo": "clash-verge-rev/clash-verge-rev",
|
||||
"file_list": [
|
||||
"Clash.Verge_{stripped_version}_amd64.deb",
|
||||
"Clash.Verge_{stripped_version}_arm64.deb"
|
||||
]
|
||||
"file_list": {
|
||||
"amd64": "Clash.Verge_{version}_amd64.deb",
|
||||
"arm64": "Clash.Verge_{version}_arm64.deb"
|
||||
}
|
||||
},
|
||||
"mihomo": {
|
||||
"repo": "MetaCubeX/mihomo",
|
||||
"file_list": [
|
||||
"mihomo-linux-amd64-compatible-{version_tag}.deb",
|
||||
"mihomo-linux-arm64-{version_tag}.deb"
|
||||
]
|
||||
"file_list": {
|
||||
"amd64": "mihomo-linux-amd64-compatible-{releases_tag}.deb",
|
||||
"arm64": "mihomo-linux-arm64-{releases_tag}.deb"
|
||||
}
|
||||
},
|
||||
"flclash": {
|
||||
"repo": "chen08209/FlClash",
|
||||
"file_list": [
|
||||
"FlClash-{stripped_version}-linux-amd64.deb"
|
||||
]
|
||||
"file_list": {
|
||||
"amd64": "FlClash-{version}-linux-amd64.deb"
|
||||
}
|
||||
},
|
||||
"hugo": {
|
||||
"repo": "gohugoio/hugo",
|
||||
"file_list": [
|
||||
"hugo_extended_{stripped_version}_linux-amd64.deb",
|
||||
"hugo_extended_{stripped_version}_linux-arm64.deb"
|
||||
]
|
||||
"file_list": {
|
||||
"amd64": "hugo_extended_{version}_linux-amd64.deb",
|
||||
"arm64": "hugo_extended_{version}_linux-arm64.deb"
|
||||
}
|
||||
},
|
||||
"rustdesk": {
|
||||
"repo": "rustdesk/rustdesk",
|
||||
"file_list": [
|
||||
"rustdesk-{version_tag}-x86_64.deb",
|
||||
"rustdesk-{version_tag}-aarch64.deb"
|
||||
]
|
||||
"file_list": {
|
||||
"amd64": "rustdesk-{releases_tag}-x86_64.deb",
|
||||
"arm64": "rustdesk-{releases_tag}-aarch64.deb"
|
||||
}
|
||||
},
|
||||
"obsidian": {
|
||||
"repo": "obsidianmd/obsidian-releases",
|
||||
"file_list": [
|
||||
"obsidian_{stripped_version}_amd64.deb"
|
||||
]
|
||||
"file_list": {
|
||||
"amd64": "obsidian_{version}_amd64.deb"
|
||||
}
|
||||
},
|
||||
"tabby": {
|
||||
"repo": "Eugeny/tabby",
|
||||
"file_list": [
|
||||
"tabby-{stripped_version}-linux-x64.deb",
|
||||
"tabby-{stripped_version}-linux-arm64.deb"
|
||||
]
|
||||
"file_list": {
|
||||
"amd64": "tabby-{version}-linux-x64.deb",
|
||||
"arm64": "tabby-{version}-linux-arm64.deb"
|
||||
}
|
||||
},
|
||||
"pandoc": {
|
||||
"repo": "jgm/pandoc",
|
||||
"file_list": [
|
||||
"pandoc-{version_tag}-1-amd64.deb",
|
||||
"pandoc-{version_tag}-1-arm64.deb"
|
||||
]
|
||||
"file_list": {
|
||||
"amd64": "pandoc-{releases_tag}-1-amd64.deb",
|
||||
"arm64": "pandoc-{releases_tag}-1-arm64.deb"
|
||||
}
|
||||
},
|
||||
"localsend": {
|
||||
"repo": "localsend/localsend",
|
||||
"file_list": [
|
||||
"LocalSend-{stripped_version}-linux-x86-64.deb",
|
||||
"LocalSend-{stripped_version}-linux-arm-64.deb"
|
||||
]
|
||||
"file_list": {
|
||||
"amd64": "LocalSend-{version}-linux-x86-64.deb",
|
||||
"arm64": "LocalSend-{version}-linux-arm-64.deb"
|
||||
}
|
||||
},
|
||||
"motrix": {
|
||||
"repo": "agalwood/Motrix",
|
||||
"file_list": [
|
||||
"Motrix_{stripped_version}_amd64.deb",
|
||||
"Motrix_{stripped_version}_arm64.deb"
|
||||
]
|
||||
"file_list": {
|
||||
"amd64": "Motrix_{version}_amd64.deb",
|
||||
"arm64": "Motrix_{version}_arm64.deb"
|
||||
}
|
||||
},
|
||||
"peazip": {
|
||||
"repo": "peazip/PeaZip",
|
||||
"file_list": [
|
||||
"peazip_{version_tag}.LINUX.GTK2-1_amd64.deb"
|
||||
]
|
||||
"file_list": {
|
||||
"amd64": "peazip_{releases_tag}.LINUX.GTK2-1_amd64.deb"
|
||||
}
|
||||
},
|
||||
"neovim": {
|
||||
"repo": "neovim/neovim-releases",
|
||||
"file_list": [
|
||||
"nvim-linux64.deb"
|
||||
]
|
||||
"file_list": {
|
||||
"amd64": "nvim-linux64.deb"
|
||||
}
|
||||
},
|
||||
"hiddify": {
|
||||
"repo": "hiddify/hiddify-app",
|
||||
"file_list": [
|
||||
"Hiddify-Debian-x64.deb"
|
||||
]
|
||||
"file_list": {
|
||||
"amd64": "Hiddify-Debian-x64.deb"
|
||||
}
|
||||
},
|
||||
"cloudflared": {
|
||||
"repo": "cloudflare/cloudflared",
|
||||
"file_list": [
|
||||
"cloudflared-linux-amd64.deb",
|
||||
"cloudflared-linux-arm64.deb"
|
||||
]
|
||||
"file_list": {
|
||||
"amd64": "cloudflared-linux-amd64.deb",
|
||||
"arm64": "cloudflared-linux-arm64.deb"
|
||||
}
|
||||
},
|
||||
"caddy": {
|
||||
"repo": "caddyserver/caddy",
|
||||
"file_list": [
|
||||
"caddy_{stripped_version}_linux_amd64.deb",
|
||||
"caddy_{stripped_version}_linux_arm64.deb"
|
||||
]
|
||||
"file_list": {
|
||||
"amd64": "caddy_{version}_linux_amd64.deb",
|
||||
"arm64": "caddy_{version}_linux_arm64.deb"
|
||||
}
|
||||
},
|
||||
"foliate": {
|
||||
"repo": "johnfactotum/foliate",
|
||||
"file_list": [
|
||||
"foliate_{version_tag}_all.deb"
|
||||
]
|
||||
"file_list": {
|
||||
"all": "foliate_{releases_tag}_all.deb"
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,106 +1,126 @@
|
||||
[
|
||||
{
|
||||
"name": "google-chrome",
|
||||
{
|
||||
"google-chrome": {
|
||||
"repo": "https://dl.google.com/linux/chrome/deb/",
|
||||
"amd64_path": "dists/stable/main/binary-amd64/Packages.gz"
|
||||
"path": {
|
||||
"amd64": "dists/stable/main/binary-amd64/Packages.gz"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "google-earth",
|
||||
"google-earth": {
|
||||
"repo": "https://dl.google.com/linux/earth/deb/",
|
||||
"amd64_path": "dists/stable/main/binary-amd64/Packages.gz"
|
||||
"path": {
|
||||
"amd64": "dists/stable/main/binary-amd64/Packages.gz"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "termius",
|
||||
"termius": {
|
||||
"repo": "https://deb.termius.com/",
|
||||
"amd64_path": "dists/squeeze/main/binary-amd64/Packages.gz"
|
||||
"path": {
|
||||
"amd64": "dists/squeeze/main/binary-amd64/Packages.gz"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "steam",
|
||||
"steam": {
|
||||
"repo": "https://repo.steampowered.com/steam/",
|
||||
"amd64_path": "dists/stable/steam/binary-amd64/Packages.gz"
|
||||
"path": {
|
||||
"amd64": "dists/stable/steam/binary-amd64/Packages.gz"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "firefox",
|
||||
"firefox": {
|
||||
"repo": "https://packages.mozilla.org/apt/",
|
||||
"amd64_path": "dists/mozilla/main/binary-amd64/Packages",
|
||||
"arm64_path": "dists/mozilla/main/binary-arm64/Packages"
|
||||
"path": {
|
||||
"amd64": "dists/mozilla/main/binary-amd64/Packages",
|
||||
"arm64": "dists/mozilla/main/binary-arm64/Packages"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "microsoft-edge",
|
||||
"microsoft-edge": {
|
||||
"repo": "https://packages.microsoft.com/repos/edge/",
|
||||
"amd64_path": "dists/stable/main/binary-amd64/Packages.gz"
|
||||
"path": {
|
||||
"amd64": "dists/stable/main/binary-amd64/Packages.gz"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "opera-stable",
|
||||
"opera-stable": {
|
||||
"repo": "https://deb.opera.com/opera-stable/",
|
||||
"amd64_path": "dists/stable/non-free/binary-amd64/Packages.gz"
|
||||
"path": {
|
||||
"amd64": "dists/stable/non-free/binary-amd64/Packages.gz"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "code",
|
||||
"code": {
|
||||
"repo": "https://packages.microsoft.com/repos/code/",
|
||||
"amd64_path": "dists/stable/main/binary-amd64/Packages.gz",
|
||||
"arm64_path": "dists/stable/main/binary-arm64/Packages.gz"
|
||||
"path": {
|
||||
"amd64": "dists/stable/main/binary-amd64/Packages.gz",
|
||||
"arm64": "dists/stable/main/binary-arm64/Packages.gz"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "tailscale",
|
||||
"tailscale": {
|
||||
"repo": "https://pkgs.tailscale.com/stable/debian/",
|
||||
"amd64_path": "dists/sid/main/binary-amd64/Packages.gz",
|
||||
"arm64_path": "dists/sid/main/binary-arm64/Packages.gz"
|
||||
"path": {
|
||||
"amd64": "dists/sid/main/binary-amd64/Packages.gz",
|
||||
"arm64": "dists/sid/main/binary-arm64/Packages.gz"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "sublime",
|
||||
"sublime": {
|
||||
"repo": "https://download.sublimetext.com/",
|
||||
"mix_path": "apt/stable/Packages.gz"
|
||||
"path": {
|
||||
"mix": "apt/stable/Packages.gz"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "black-desk",
|
||||
"black-desk": {
|
||||
"repo": "https://github.com/black-desk/debs/releases/latest/download/",
|
||||
"amd64_path": "Packages"
|
||||
"path": {
|
||||
"amd64": "Packages"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "typora",
|
||||
"typora": {
|
||||
"repo": "https://typoraio.cn/linux/",
|
||||
"mix_path": "Packages.gz"
|
||||
"path": {
|
||||
"mix": "Packages.gz"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "zotero",
|
||||
"zotero": {
|
||||
"repo": "https://zotero.retorque.re/file/apt-package-archive/",
|
||||
"amd64_path": "Packages"
|
||||
"path": {
|
||||
"amd64": "Packages"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "gh",
|
||||
"gh": {
|
||||
"repo": "https://cli.github.com/packages/",
|
||||
"amd64_path": "dists/stable/main/binary-amd64/Packages.gz",
|
||||
"arm64_path": "dists/stable/main/binary-arm64/Packages.gz"
|
||||
"path": {
|
||||
"amd64": "dists/stable/main/binary-amd64/Packages.gz",
|
||||
"arm64": "dists/stable/main/binary-arm64/Packages.gz"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "dufs",
|
||||
"dufs": {
|
||||
"repo": "https://github.com/wcbing-build/dufs-debs/releases/latest/download/",
|
||||
"mix_path": "Packages"
|
||||
"path": {
|
||||
"mix": "Packages"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "frp",
|
||||
"frp": {
|
||||
"repo": "https://github.com/wcbing-build/frp-debs/releases/latest/download/",
|
||||
"mix_path": "Packages"
|
||||
"path": {
|
||||
"mix": "Packages"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "lazydocker",
|
||||
"lazydocker": {
|
||||
"repo": "https://github.com/wcbing-build/lazydocker-debs/releases/latest/download/",
|
||||
"mix_path": "Packages"
|
||||
"path": {
|
||||
"mix": "Packages"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "lazygit",
|
||||
"lazygit": {
|
||||
"repo": "https://github.com/wcbing-build/lazygit-debs/releases/latest/download/",
|
||||
"mix_path": "Packages"
|
||||
"path": {
|
||||
"mix": "Packages"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "nexttrace",
|
||||
"nexttrace": {
|
||||
"repo": "https://github.com/nxtrace/nexttrace-debs/releases/latest/download/",
|
||||
"mix_path": "Packages"
|
||||
"path": {
|
||||
"mix": "Packages"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "debiancn",
|
||||
"debiancn": {
|
||||
"repo": "https://mirrors.cernet.edu.cn/debiancn/",
|
||||
"amd64_path": "dists/bookworm/main/binary-amd64/Packages.gz"
|
||||
"path": {
|
||||
"amd64": "dists/bookworm/main/binary-amd64/Packages.gz"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
61
get-github-releases.py
Executable file
61
get-github-releases.py
Executable file
@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import requests
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from concurrent.futures import ThreadPoolExecutor, wait
|
||||
from check_downloader import check_download
|
||||
|
||||
github_info_list = {}
|
||||
|
||||
CONFIG = {"data_dir": "data", "proxy": "", "thread": 5}
|
||||
|
||||
|
||||
# 读取命令行参数
|
||||
def read_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("-d", "--data", default="data", help="从 <DATA> 读取仓库配置")
|
||||
parser.add_argument(
|
||||
"-p", "--proxy", default="", help="Github 代理，<PROXY> 必须以 / 结尾"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-t", "--thread", type=int, default=5, help="并发下载线程数量，默认为 5"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
CONFIG.update({"data_dir": args.data, "proxy": args.proxy, "thread": args.thread})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
read_args()
|
||||
|
||||
# read all repo info 读取所有仓库配置
|
||||
with open(os.path.join(CONFIG["data_dir"], "github.json"), "r") as f:
|
||||
github_info_list = json.load(f)
|
||||
|
||||
tasks = []
|
||||
with ThreadPoolExecutor(max_workers=CONFIG["thread"]) as executor:
|
||||
for name, repo in github_info_list.items():
|
||||
release_url = (
|
||||
f'{CONFIG["proxy"]}https://github.com/{repo["repo"]}/releases'
|
||||
)
|
||||
# get latest releases tag 获取最新版本标签
|
||||
location = requests.head(release_url + "/latest").headers.get("Location", "")
|
||||
match = re.search(r".*releases/tag/([^/]+)", location)
|
||||
if not match:
|
||||
continue
|
||||
releases_tag = match.group(1)
|
||||
version = match.group() if (match := re.search("[0-9].*", releases_tag)) else ""
|
||||
|
||||
|
||||
for arch, file_name in repo["file_list"].items():
|
||||
release_file = file_name.format(
|
||||
releases_tag=releases_tag, version=version
|
||||
)
|
||||
file_url = f"{release_url}/download/{releases_tag}/{release_file}"
|
||||
# 提交任务到线程池
|
||||
tasks.append(
|
||||
executor.submit(check_download, name, version, file_url, arch)
|
||||
)
|
||||
# 等待所有任务完成
|
||||
wait(tasks)
|
||||
19
init-deb.py
Executable file
19
init-deb.py
Executable file
@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import sqlite3
|
||||
|
||||
# create table
|
||||
conn = sqlite3.connect("data/deb.db")
|
||||
for arch in ["amd64", "arm64", "all"]:
|
||||
conn.execute(
|
||||
f"""
|
||||
CREATE TABLE IF NOT EXISTS '{arch}' (
|
||||
name TEXT UNIQUE,
|
||||
version TEXT,
|
||||
url TEXT
|
||||
);
|
||||
"""
|
||||
)
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
27
init_deb.py
27
init_deb.py
@ -1,27 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import sqlite3
|
||||
|
||||
# create table
|
||||
conn = sqlite3.connect("data/deb.db")
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS x86_64 (
|
||||
name TEXT UNIQUE,
|
||||
version TEXT,
|
||||
url TEXT
|
||||
);
|
||||
"""
|
||||
)
|
||||
conn.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS arm64 (
|
||||
name TEXT UNIQUE,
|
||||
version TEXT,
|
||||
url TEXT
|
||||
);
|
||||
"""
|
||||
)
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
@ -17,30 +17,37 @@ package_version = {arch: {} for arch in ["all", "amd64", "i386", "arm64"]}
|
||||
package_info = {arch: {} for arch in ["all", "amd64", "i386", "arm64"]}
|
||||
lock = {arch: Lock() for arch in ["all", "amd64", "i386", "arm64"]}
|
||||
|
||||
USER_AGENT = "Debian APT-HTTP/1.3 (2.6.1)" # from Debian 12
|
||||
|
||||
"""
|
||||
repo info json format:
|
||||
{
|
||||
"name": repo name
|
||||
"repo_name": {
|
||||
"repo": repo url, end with "/"
|
||||
"xxx_path": repo xxx Packages file path, start with no "/"
|
||||
"xxx_path": {
|
||||
"arch": repo Packages file path of "arch", start with no "/"
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
def read_repo_list(repo_list_file):
|
||||
def read_repo_list(repo_list_file: str) -> dict:
|
||||
try:
|
||||
with open(repo_list_file, "r") as f:
|
||||
return json.load(f)
|
||||
except Exception as e:
|
||||
logging.error(f"Error reading repo list: {e}")
|
||||
return []
|
||||
return {}
|
||||
|
||||
|
||||
# get the packages file content from remote repo
|
||||
def get_remote_packages(repo_url, file_path):
|
||||
def get_remote_packages(repo_url: str, file_path: str) -> bytes:
|
||||
"""
|
||||
get the packages file content from remote repo
|
||||
"""
|
||||
file_url = repo_url + file_path
|
||||
try:
|
||||
response = requests.get(file_url, timeout=10)
|
||||
response = requests.get(
|
||||
file_url, timeout=10, headers={"User-Agent": USER_AGENT}
|
||||
)
|
||||
if response.status_code != 200:
|
||||
logging.error(
|
||||
f"GetError: {file_url} returned status {response.status_code}"
|
||||
@ -67,9 +74,11 @@ def get_remote_packages(repo_url, file_path):
|
||||
return b""
|
||||
|
||||
|
||||
def get_latest(deb_packages):
|
||||
# split the information of each packet, store it in infoList
|
||||
# 将每个包的信息分割开，存放到 infoList 中
|
||||
def get_latest(deb_packages: bytes):
|
||||
"""
|
||||
split the information of each packet, deduplication and store the latest in infoList
|
||||
将每个包的信息分割开，去重并将最新的存放到 infoList 中
|
||||
"""
|
||||
deb_packages = re.sub(rb"^Package: ", b"{{start}}Package: ", deb_packages, flags=re.MULTILINE)
|
||||
info_list = deb_packages.split(b"{{start}}")[1:]
|
||||
|
||||
@ -97,18 +106,14 @@ def get_latest(deb_packages):
|
||||
return
|
||||
|
||||
|
||||
def process_repo(r):
|
||||
def process_repo(r: dict):
|
||||
"""
|
||||
获取仓库中不同架构子仓库的内容，最后调用 get_latest 去重并保存。
|
||||
"""
|
||||
try:
|
||||
deb_packages = b""
|
||||
if r.get("mix_path"): # 获取扁平 Repo 中包信息
|
||||
deb_packages += get_remote_packages(r["repo"], r["mix_path"])
|
||||
else:
|
||||
if r.get("amd64_path"): # 获取 Repo 中 Amd64 包信息
|
||||
deb_packages += get_remote_packages(r["repo"], r["amd64_path"])
|
||||
if r.get("arm64_path"): # 获取 Repo 中 Arm64 包信息
|
||||
deb_packages += get_remote_packages(r["repo"], r["arm64_path"])
|
||||
if r.get("all_path"): # 获取 Repo 中 All 包信息
|
||||
deb_packages += get_remote_packages(r["repo"], r["all_path"])
|
||||
for arch, path in r["path"].items():
|
||||
deb_packages += get_remote_packages(r["repo"], path)
|
||||
get_latest(deb_packages)
|
||||
except Exception as e:
|
||||
logging.error(f"Error processing repo {r.get('name', 'unknown')}: {e}")
|
||||
@ -137,19 +142,19 @@ if __name__ == "__main__":
|
||||
with open(args.local) as f:
|
||||
get_latest(f.read().encode())
|
||||
|
||||
repo_list_file = args.repo
|
||||
repo_list = read_repo_list(repo_list_file)
|
||||
# 读取 repo_list 配置
|
||||
repo_list = read_repo_list(args.repo)
|
||||
if not repo_list:
|
||||
sys.exit()
|
||||
|
||||
# 多线程，同时限制最大线程数
|
||||
with ThreadPoolExecutor(max_workers=10) as executor:
|
||||
executor.map(process_repo, repo_list)
|
||||
executor.map(process_repo, repo_list.values())
|
||||
|
||||
# 分别输出到不同文件
|
||||
os.makedirs("deb/amd64/", exist_ok=True)
|
||||
os.makedirs("deb/arm64/", exist_ok=True)
|
||||
|
||||
# 分别输出到不同文件
|
||||
with open("deb/amd64/Packages", "+wb") as f:
|
||||
for i in package_info["amd64"].values():
|
||||
f.write(i)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user