refactor: save packages info to file

check_downloader.py

@@ -4,9 +4,11 @@ import os
 import sqlite3
 import sys
 import logging
+import re
 from threading import Lock
 
-BASE_DIR = "deb"
+DEB_BASE_DIR = "deb"
+PACKAGES_DIR = "packages"
 DB_DIR = "data"
 USER_AGENT = "Debian APT-HTTP/1.3 (2.6.1)"  # from Debian 12
 
@@ -19,43 +21,78 @@ logging.basicConfig(
 )
 
 
-def download(url):
+def download(url, file_path) -> bool:
     """Download file using curl with APT User-Agent."""
-    file_path = os.path.join(BASE_DIR, url.split("?")[0])
-    os.makedirs(os.path.dirname(file_path), exist_ok=True)
-    subprocess.run(["curl", "-H", f"User-Agent: {USER_AGENT}", "-fsLo", file_path, url])
+    curl_process = subprocess.run(
+        ["curl", "-H", f"User-Agent: {USER_AGENT}", "-fsLo", file_path, url]
+    )
+    if curl_process.returncode or not os.path.exists(file_path):
+        logging.error(f"Failed to download {url}")
+        return False
+    return True
+
+
+def scan(name, arch, url, file_path) -> bool:
+    scan_process = subprocess.run(
+        ["apt-ftparchive", "packages", file_path], capture_output=True
+    )
+    package = scan_process.stdout.decode()
+    package = re.sub(
+        r"^(Filename: ).*", f"\\1{url}", package, flags=re.MULTILINE
+    )  # 替换 Filename 开头的行
+
+    package_file_path = os.path.join(PACKAGES_DIR, arch, f"{name}.package")
+
+    try:
+        with open(package_file_path, "w") as f:
+            f.write(package)
+            return True
+    except IOError as e:
+        logging.error(f"Failed to write package file for {name}: {e}")
+        return False
 
 
 def check_download(name, version, url, arch):
     """Check and handle package download/update."""
     logging.info("%s:%s = %s", name, arch, version)
 
-    db_path = os.path.join("data", f"{BASE_DIR}.db")
+    file_path = os.path.join(DEB_BASE_DIR, arch, f"{name}_{version}_{arch}.deb")
+    local_version = None
+    db_path = os.path.join("data", f"{DEB_BASE_DIR}.db")
     # get local version
     with version_lock, sqlite3.connect(db_path) as conn:
         res = conn.execute(
-            f"SELECT version, url FROM '{arch}' WHERE name = ?", (name,)
+            f"SELECT version FROM '{arch}' WHERE name = ?", (name,)
         ).fetchone()
     if res:
-        local_version, local_url = res
-        if local_version != version:
-            print(f"Update: {name}:{arch} ({local_version} -> {version})")
-            download(url)
-            # update database
-            with version_lock, sqlite3.connect(db_path) as conn:
-                conn.execute(
-                    f"UPDATE '{arch}' SET version = ?, url = ? WHERE name = ?",
-                    (version, url, name),
-                )
-                conn.commit()
-            # remove old version
-            if local_url != url:  # 防止固定下载链接
-                old_file_path = os.path.join(BASE_DIR, local_url.split("?")[0])
-                if os.path.exists(old_file_path):
-                    os.remove(old_file_path)
+        local_version = res[0]
+        if local_version == version:
+            return
+
+    # download and scan
+    logging.info(f"Downloading {name}:{arch} ({version})")
+    os.makedirs(os.path.join(DEB_BASE_DIR, arch), exist_ok=True)
+    if not download(url, file_path):
+        return
+    os.makedirs(os.path.join(PACKAGES_DIR, arch), exist_ok=True)
+    if not scan(name, arch, url, file_path):
+        return
+
+    if res:
+        print(f"Update: {name}:{arch} ({local_version} -> {version})")
+        # update database
+        with version_lock, sqlite3.connect(db_path) as conn:
+            conn.execute(
+                f"UPDATE '{arch}' SET version = ?, url = ? WHERE name = ?",
+                (version, url, name),
+            )
+            conn.commit()
+        # remove old version
+        old_file_path = os.path.join(DEB_BASE_DIR, f"{name}_{local_version}_{arch}.deb")
+        if os.path.exists(old_file_path):
+            os.remove(old_file_path)
     else:
         print(f"AddNew: {name}:{arch} ({version})")
-        download(url)
         # update database
         with version_lock, sqlite3.connect(db_path) as conn:
             conn.execute(

run.sh

@@ -7,15 +7,8 @@ find get -type f -name "*.sh" -exec sh {} \;
 # generate the html
 ./gen-list-html.py
 
-# generate the Packages file
-## generate the local Packages file
-cd deb
-apt-ftparchive packages . > tmpPackages
-sed -i "s|\./\(https\?\):/|\1://|g" tmpPackages
-cd ..
-
 ## merge the Packages file from local package
-cat $(find packages -name "*.package") >> deb/tmpPackages
+cat $(find packages -name "*.package") > deb/tmpPackages
 
 ## merge the Packages files from third-party repositories
 ./merge-apt-repo.py --local deb/tmpPackages