Created
December 2, 2025 03:16
-
-
Save asarkar/ea6d5652ce90efe26915323b6176d5c6 to your computer and use it in GitHub Desktop.
list extras (Provides-Extra) for PyPI packages
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| """ | |
| list_extras.py - list extras (Provides-Extra) for PyPI packages (parallel fetching) | |
| Usage: | |
| ./list_extras.py pkg1 [pkg2 ...] | |
| ./list_extras.py django-stubs requests | |
| ./list_extras.py django-stubs==2.6.0 requests==2.31.0 | |
| Version selection is only via "pkg==version". | |
| """ | |
| import argparse | |
| import io | |
| import json | |
| import tarfile | |
| import urllib.request | |
| import zipfile | |
| from concurrent.futures import ThreadPoolExecutor | |
| from typing import Any, cast | |
| PYPI_JSON = "https://pypi.org/pypi/{pkg}/json" | |
| def fetch_json(pkg: str) -> dict[str, Any]: | |
| with urllib.request.urlopen(PYPI_JSON.format(pkg=pkg)) as r: | |
| return cast(dict[str, Any], json.load(r)) | |
| def choose_release_file(files: list[dict[str, Any]]) -> dict[str, Any] | None: | |
| wheels = [f for f in files if f["filename"].endswith(".whl")] | |
| if wheels: | |
| return wheels[0] | |
| sdists = [ | |
| f | |
| for f in files | |
| if f["filename"].endswith((".tar.gz", ".zip", ".tar.bz2", ".tgz")) | |
| ] | |
| if sdists: | |
| return sdists[0] | |
| return files[0] if files else None | |
| def read_metadata_from_wheel(b: bytes) -> str | None: | |
| with zipfile.ZipFile(io.BytesIO(b)) as z: | |
| for n in z.namelist(): | |
| if n.endswith("METADATA"): | |
| return z.read(n).decode(errors="replace") | |
| return None | |
| def read_metadata_from_sdist(b: bytes, filename: str) -> str | None: | |
| if filename.endswith(".zip"): | |
| with zipfile.ZipFile(io.BytesIO(b)) as z: | |
| for n in z.namelist(): | |
| if n.endswith(("PKG-INFO", "METADATA")): | |
| return z.read(n).decode(errors="replace") | |
| return None | |
| try: | |
| with tarfile.open(fileobj=io.BytesIO(b), mode="r:*") as t: | |
| for m in t.getmembers(): | |
| if m.isfile() and m.name.endswith(("PKG-INFO", "METADATA")): | |
| f = t.extractfile(m) | |
| return f.read().decode(errors="replace") if f else None | |
| except tarfile.ReadError: | |
| return None | |
| return None | |
| def extract_extras(meta_text: str) -> list[str]: | |
| return [ | |
| line.split(":", 1)[1].strip() | |
| for line in meta_text.splitlines() | |
| if line.startswith("Provides-Extra:") | |
| ] | |
| def inspect_package(pkg: str) -> dict[str, Any]: | |
| # parse pkg==version | |
| version: str | None = None | |
| if "==" in pkg: | |
| pkg, version = pkg.split("==", 1) | |
| result: dict[str, Any] = { | |
| "package": pkg, | |
| "version": None, | |
| "extras": [], | |
| "error": None, | |
| } | |
| try: | |
| data = fetch_json(pkg) | |
| except Exception as e: | |
| result["error"] = f"Failed to fetch metadata: {e}" | |
| return result | |
| if not version: | |
| # data is dict[str, Any], so .get is allowed | |
| version = data.get("info", {}).get("version") # type: ignore[assignment] | |
| result["version"] = version | |
| files = data.get("releases", {}).get(version) or [] | |
| sel = choose_release_file(files) | |
| if not sel: | |
| result["error"] = f"No release files for {pkg}=={version}" | |
| return result | |
| try: | |
| with urllib.request.urlopen(sel["url"]) as r: | |
| b = r.read() | |
| except Exception as e: | |
| result["error"] = f"Failed to download file: {e}" | |
| return result | |
| if sel["filename"].endswith(".whl"): | |
| meta = read_metadata_from_wheel(b) | |
| else: | |
| meta = read_metadata_from_sdist(b, sel["filename"]) | |
| if not meta: | |
| result["error"] = "Could not read METADATA or PKG-INFO." | |
| return result | |
| result["extras"] = extract_extras(meta) | |
| return result | |
| def print_results(results: list[dict[str, Any]]) -> None: | |
| for r in results: | |
| pkg = r["package"] | |
| ver = r["version"] | |
| print(f"Extras for {pkg}=={ver}:") | |
| if r["error"]: | |
| # Still follow the same format, comment-style | |
| print(f" # ERROR: {r['error']}") | |
| continue | |
| for e in r["extras"]: | |
| print(f" - {e}") | |
| def main() -> int: | |
| p = argparse.ArgumentParser(description="List extras (Provides-Extra) for PyPI packages.") | |
| p.add_argument("packages", nargs="+", help="one or more packages (optionally pkg==version)") | |
| args = p.parse_args() | |
| pkgs = args.packages | |
| max_workers = min(32, len(pkgs)) or 1 | |
| with ThreadPoolExecutor(max_workers=max_workers) as ex: | |
| results = list(ex.map(inspect_package, pkgs)) | |
| print_results(results) | |
| return 1 if any(r["error"] for r in results) else 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment